From 0ebf5bdf043a27fd3dfb7f92e0cb63d88954c44d Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 03:47:29 +0200 Subject: Adding upstream version 115.8.0esr. Signed-off-by: Daniel Baumann --- .../libwebrtc/modules/audio_coding/BUILD.gn | 1805 +++++++++++++++++ third_party/libwebrtc/modules/audio_coding/DEPS | 7 + third_party/libwebrtc/modules/audio_coding/OWNERS | 4 + .../modules/audio_coding/acm2/acm_receive_test.cc | 161 ++ .../modules/audio_coding/acm2/acm_receive_test.h | 98 + .../modules/audio_coding/acm2/acm_receiver.cc | 352 ++++ .../modules/audio_coding/acm2/acm_receiver.h | 234 +++ .../audio_coding/acm2/acm_receiver_unittest.cc | 455 +++++ .../modules/audio_coding/acm2/acm_remixing.cc | 114 ++ .../modules/audio_coding/acm2/acm_remixing.h | 34 + .../audio_coding/acm2/acm_remixing_unittest.cc | 191 ++ .../modules/audio_coding/acm2/acm_resampler.cc | 61 + .../modules/audio_coding/acm2/acm_resampler.h | 41 + .../modules/audio_coding/acm2/acm_send_test.cc | 174 ++ .../modules/audio_coding/acm2/acm_send_test.h | 91 + .../audio_coding/acm2/audio_coding_module.cc | 637 ++++++ .../acm2/audio_coding_module_unittest.cc | 1278 ++++++++++++ .../modules/audio_coding/acm2/call_statistics.cc | 63 + .../modules/audio_coding/acm2/call_statistics.h | 64 + .../audio_coding/acm2/call_statistics_unittest.cc | 57 + .../modules/audio_coding/audio_coding.gni | 25 + .../modules/audio_coding/audio_coding_gn/moz.build | 236 +++ .../audio_coding_module_typedefs_gn/moz.build | 201 ++ .../audio_coding_opus_common_gn/moz.build | 225 ++ .../audio_coding/audio_encoder_cng_gn/moz.build | 232 +++ .../audio_network_adaptor_config.cc | 34 + .../audio_network_adaptor_impl.cc | 171 ++ .../audio_network_adaptor_impl.h | 89 + .../audio_network_adaptor_impl_unittest.cc | 306 +++ .../audio_network_adaptor/bitrate_controller.cc | 73 + .../audio_network_adaptor/bitrate_controller.h | 59 + .../bitrate_controller_unittest.cc | 236 +++ .../audio_network_adaptor/channel_controller.cc | 62 + .../audio_network_adaptor/channel_controller.h | 58 + .../channel_controller_unittest.cc | 101 + .../audio_network_adaptor/config.proto | 196 ++ .../audio_network_adaptor/controller.cc | 19 + .../audio_network_adaptor/controller.h | 42 + .../audio_network_adaptor/controller_manager.cc | 454 +++++ .../audio_network_adaptor/controller_manager.h | 124 ++ .../controller_manager_unittest.cc | 486 +++++ .../audio_network_adaptor/debug_dump.proto | 42 + .../audio_network_adaptor/debug_dump_writer.cc | 163 ++ .../audio_network_adaptor/debug_dump_writer.h | 54 + .../audio_network_adaptor/dtx_controller.cc | 51 + .../audio_network_adaptor/dtx_controller.h | 52 + .../dtx_controller_unittest.cc | 82 + .../audio_network_adaptor/event_log_writer.cc | 78 + .../audio_network_adaptor/event_log_writer.h | 44 + .../event_log_writer_unittest.cc | 240 +++ .../fec_controller_plr_based.cc | 113 ++ .../fec_controller_plr_based.h | 74 + .../fec_controller_plr_based_unittest.cc | 489 +++++ .../frame_length_controller.cc | 201 ++ .../frame_length_controller.h | 93 + .../frame_length_controller_unittest.cc | 444 ++++ .../frame_length_controller_v2.cc | 73 + .../frame_length_controller_v2.h | 44 + .../frame_length_controller_v2_unittest.cc | 121 ++ .../include/audio_network_adaptor.h | 49 + .../include/audio_network_adaptor_config.h | 51 + .../mock/mock_audio_network_adaptor.h | 57 + .../audio_network_adaptor/mock/mock_controller.h | 35 + .../mock/mock_controller_manager.h | 34 + .../mock/mock_debug_dump_writer.h | 44 + .../audio_network_adaptor/parse_ana_dump.py | 149 ++ .../audio_network_adaptor/util/threshold_curve.h | 118 ++ .../util/threshold_curve_unittest.cc | 632 ++++++ .../audio_network_adaptor_config_gn/moz.build | 217 ++ .../audio_network_adaptor_gn/moz.build | 242 +++ .../modules/audio_coding/codecs/audio_decoder.h | 20 + .../modules/audio_coding/codecs/audio_encoder.h | 20 + .../builtin_audio_decoder_factory_unittest.cc | 168 ++ .../builtin_audio_encoder_factory_unittest.cc | 178 ++ .../audio_coding/codecs/cng/audio_encoder_cng.cc | 322 +++ .../audio_coding/codecs/cng/audio_encoder_cng.h | 49 + .../codecs/cng/audio_encoder_cng_unittest.cc | 520 +++++ .../audio_coding/codecs/cng/cng_unittest.cc | 252 +++ .../modules/audio_coding/codecs/cng/webrtc_cng.cc | 436 ++++ .../modules/audio_coding/codecs/cng/webrtc_cng.h | 99 + .../audio_coding/codecs/g711/audio_decoder_pcm.cc | 102 + .../audio_coding/codecs/g711/audio_decoder_pcm.h | 81 + .../audio_coding/codecs/g711/audio_encoder_pcm.cc | 126 ++ .../audio_coding/codecs/g711/audio_encoder_pcm.h | 128 ++ .../audio_coding/codecs/g711/g711_interface.c | 59 + .../audio_coding/codecs/g711/g711_interface.h | 136 ++ .../audio_coding/codecs/g711/test/testG711.cc | 168 ++ .../audio_coding/codecs/g722/audio_decoder_g722.cc | 178 ++ .../audio_coding/codecs/g722/audio_decoder_g722.h | 86 + .../audio_coding/codecs/g722/audio_encoder_g722.cc | 156 ++ .../audio_coding/codecs/g722/audio_encoder_g722.h | 71 + .../audio_coding/codecs/g722/g722_interface.c | 104 + .../audio_coding/codecs/g722/g722_interface.h | 174 ++ .../audio_coding/codecs/g722/test/testG722.cc | 155 ++ .../modules/audio_coding/codecs/ilbc/abs_quant.c | 82 + .../modules/audio_coding/codecs/ilbc/abs_quant.h | 42 + .../audio_coding/codecs/ilbc/abs_quant_loop.c | 89 + .../audio_coding/codecs/ilbc/abs_quant_loop.h | 36 + .../audio_coding/codecs/ilbc/audio_decoder_ilbc.cc | 110 + .../audio_coding/codecs/ilbc/audio_decoder_ilbc.h | 54 + .../audio_coding/codecs/ilbc/audio_encoder_ilbc.cc | 151 ++ .../audio_coding/codecs/ilbc/audio_encoder_ilbc.h | 61 + .../audio_coding/codecs/ilbc/augmented_cb_corr.c | 64 + .../audio_coding/codecs/ilbc/augmented_cb_corr.h | 42 + .../modules/audio_coding/codecs/ilbc/bw_expand.c | 44 + .../modules/audio_coding/codecs/ilbc/bw_expand.h | 37 + .../audio_coding/codecs/ilbc/cb_construct.c | 80 + .../audio_coding/codecs/ilbc/cb_construct.h | 44 + .../audio_coding/codecs/ilbc/cb_mem_energy.c | 81 + .../audio_coding/codecs/ilbc/cb_mem_energy.h | 37 + .../codecs/ilbc/cb_mem_energy_augmentation.c | 69 + .../codecs/ilbc/cb_mem_energy_augmentation.h | 34 + .../audio_coding/codecs/ilbc/cb_mem_energy_calc.c | 67 + .../audio_coding/codecs/ilbc/cb_mem_energy_calc.h | 36 + .../modules/audio_coding/codecs/ilbc/cb_search.c | 405 ++++ .../modules/audio_coding/codecs/ilbc/cb_search.h | 40 + .../audio_coding/codecs/ilbc/cb_search_core.c | 115 ++ .../audio_coding/codecs/ilbc/cb_search_core.h | 41 + .../codecs/ilbc/cb_update_best_index.c | 89 + .../codecs/ilbc/cb_update_best_index.h | 39 + .../modules/audio_coding/codecs/ilbc/chebyshev.c | 76 + .../modules/audio_coding/codecs/ilbc/chebyshev.h | 38 + .../modules/audio_coding/codecs/ilbc/comp_corr.c | 51 + .../modules/audio_coding/codecs/ilbc/comp_corr.h | 39 + .../audio_coding/codecs/ilbc/complexityMeasures.m | 57 + .../modules/audio_coding/codecs/ilbc/constants.c | 667 ++++++ .../modules/audio_coding/codecs/ilbc/constants.h | 95 + .../codecs/ilbc/create_augmented_vec.c | 83 + .../codecs/ilbc/create_augmented_vec.h | 38 + .../modules/audio_coding/codecs/ilbc/decode.c | 261 +++ .../modules/audio_coding/codecs/ilbc/decode.h | 42 + .../audio_coding/codecs/ilbc/decode_residual.c | 185 ++ .../audio_coding/codecs/ilbc/decode_residual.h | 45 + .../codecs/ilbc/decoder_interpolate_lsf.c | 85 + .../codecs/ilbc/decoder_interpolate_lsf.h | 41 + .../modules/audio_coding/codecs/ilbc/defines.h | 225 ++ .../modules/audio_coding/codecs/ilbc/do_plc.c | 309 +++ .../modules/audio_coding/codecs/ilbc/do_plc.h | 44 + .../modules/audio_coding/codecs/ilbc/encode.c | 517 +++++ .../modules/audio_coding/codecs/ilbc/encode.h | 38 + .../audio_coding/codecs/ilbc/energy_inverse.c | 46 + .../audio_coding/codecs/ilbc/energy_inverse.h | 36 + .../audio_coding/codecs/ilbc/enh_upsample.c | 112 + .../audio_coding/codecs/ilbc/enh_upsample.h | 33 + .../modules/audio_coding/codecs/ilbc/enhancer.c | 53 + .../modules/audio_coding/codecs/ilbc/enhancer.h | 40 + .../audio_coding/codecs/ilbc/enhancer_interface.c | 382 ++++ .../audio_coding/codecs/ilbc/enhancer_interface.h | 36 + .../audio_coding/codecs/ilbc/filtered_cb_vecs.c | 50 + .../audio_coding/codecs/ilbc/filtered_cb_vecs.h | 39 + .../audio_coding/codecs/ilbc/frame_classify.c | 90 + .../audio_coding/codecs/ilbc/frame_classify.h | 34 + .../audio_coding/codecs/ilbc/gain_dequant.c | 47 + .../audio_coding/codecs/ilbc/gain_dequant.h | 36 + .../modules/audio_coding/codecs/ilbc/gain_quant.c | 105 + .../modules/audio_coding/codecs/ilbc/gain_quant.h | 36 + .../modules/audio_coding/codecs/ilbc/get_cd_vec.c | 126 ++ .../modules/audio_coding/codecs/ilbc/get_cd_vec.h | 40 + .../audio_coding/codecs/ilbc/get_lsp_poly.c | 84 + .../audio_coding/codecs/ilbc/get_lsp_poly.h | 46 + .../audio_coding/codecs/ilbc/get_sync_seq.c | 111 + .../audio_coding/codecs/ilbc/get_sync_seq.h | 41 + .../modules/audio_coding/codecs/ilbc/hp_input.c | 90 + .../modules/audio_coding/codecs/ilbc/hp_input.h | 38 + .../modules/audio_coding/codecs/ilbc/hp_output.c | 91 + .../modules/audio_coding/codecs/ilbc/hp_output.h | 38 + .../modules/audio_coding/codecs/ilbc/ilbc.c | 288 +++ .../modules/audio_coding/codecs/ilbc/ilbc.h | 251 +++ .../audio_coding/codecs/ilbc/ilbc_unittest.cc | 140 ++ .../audio_coding/codecs/ilbc/index_conv_dec.c | 40 + .../audio_coding/codecs/ilbc/index_conv_dec.h | 27 + .../audio_coding/codecs/ilbc/index_conv_enc.c | 45 + .../audio_coding/codecs/ilbc/index_conv_enc.h | 31 + .../modules/audio_coding/codecs/ilbc/init_decode.c | 98 + .../modules/audio_coding/codecs/ilbc/init_decode.h | 38 + .../modules/audio_coding/codecs/ilbc/init_encode.c | 73 + .../modules/audio_coding/codecs/ilbc/init_encode.h | 36 + .../modules/audio_coding/codecs/ilbc/interpolate.c | 48 + .../modules/audio_coding/codecs/ilbc/interpolate.h | 35 + .../audio_coding/codecs/ilbc/interpolate_samples.c | 53 + .../audio_coding/codecs/ilbc/interpolate_samples.h | 35 + .../modules/audio_coding/codecs/ilbc/lpc_encode.c | 62 + .../modules/audio_coding/codecs/ilbc/lpc_encode.h | 42 + .../modules/audio_coding/codecs/ilbc/lsf_check.c | 73 + .../modules/audio_coding/codecs/ilbc/lsf_check.h | 32 + .../codecs/ilbc/lsf_interpolate_to_poly_dec.c | 44 + .../codecs/ilbc/lsf_interpolate_to_poly_dec.h | 37 + .../codecs/ilbc/lsf_interpolate_to_poly_enc.c | 48 + .../codecs/ilbc/lsf_interpolate_to_poly_enc.h | 38 + .../modules/audio_coding/codecs/ilbc/lsf_to_lsp.c | 63 + .../modules/audio_coding/codecs/ilbc/lsf_to_lsp.h | 34 + .../modules/audio_coding/codecs/ilbc/lsf_to_poly.c | 88 + .../modules/audio_coding/codecs/ilbc/lsf_to_poly.h | 33 + .../modules/audio_coding/codecs/ilbc/lsp_to_lsf.c | 86 + .../modules/audio_coding/codecs/ilbc/lsp_to_lsf.h | 35 + .../modules/audio_coding/codecs/ilbc/my_corr.c | 56 + .../modules/audio_coding/codecs/ilbc/my_corr.h | 36 + .../audio_coding/codecs/ilbc/nearest_neighbor.c | 35 + .../audio_coding/codecs/ilbc/nearest_neighbor.h | 37 + .../modules/audio_coding/codecs/ilbc/pack_bits.c | 253 +++ .../modules/audio_coding/codecs/ilbc/pack_bits.h | 34 + .../modules/audio_coding/codecs/ilbc/poly_to_lsf.c | 32 + .../modules/audio_coding/codecs/ilbc/poly_to_lsf.h | 32 + .../modules/audio_coding/codecs/ilbc/poly_to_lsp.c | 159 ++ .../modules/audio_coding/codecs/ilbc/poly_to_lsp.h | 36 + .../modules/audio_coding/codecs/ilbc/refiner.c | 141 ++ .../modules/audio_coding/codecs/ilbc/refiner.h | 44 + .../codecs/ilbc/simple_interpolate_lsf.c | 133 ++ .../codecs/ilbc/simple_interpolate_lsf.h | 48 + .../audio_coding/codecs/ilbc/simple_lpc_analysis.c | 96 + .../audio_coding/codecs/ilbc/simple_lpc_analysis.h | 37 + .../audio_coding/codecs/ilbc/simple_lsf_dequant.c | 62 + .../audio_coding/codecs/ilbc/simple_lsf_dequant.h | 34 + .../audio_coding/codecs/ilbc/simple_lsf_quant.c | 49 + .../audio_coding/codecs/ilbc/simple_lsf_quant.h | 37 + .../modules/audio_coding/codecs/ilbc/smooth.c | 212 ++ .../modules/audio_coding/codecs/ilbc/smooth.h | 35 + .../audio_coding/codecs/ilbc/smooth_out_data.c | 56 + .../audio_coding/codecs/ilbc/smooth_out_data.h | 33 + .../modules/audio_coding/codecs/ilbc/sort_sq.c | 53 + .../modules/audio_coding/codecs/ilbc/sort_sq.h | 36 + .../modules/audio_coding/codecs/ilbc/split_vq.c | 63 + .../modules/audio_coding/codecs/ilbc/split_vq.h | 38 + .../audio_coding/codecs/ilbc/state_construct.c | 116 ++ .../audio_coding/codecs/ilbc/state_construct.h | 38 + .../audio_coding/codecs/ilbc/state_search.c | 121 ++ .../audio_coding/codecs/ilbc/state_search.h | 41 + .../modules/audio_coding/codecs/ilbc/swap_bytes.c | 35 + .../modules/audio_coding/codecs/ilbc/swap_bytes.h | 35 + .../modules/audio_coding/codecs/ilbc/test/empty.cc | 0 .../audio_coding/codecs/ilbc/test/iLBC_test.c | 238 +++ .../audio_coding/codecs/ilbc/test/iLBC_testLib.c | 215 ++ .../codecs/ilbc/test/iLBC_testprogram.c | 343 ++++ .../modules/audio_coding/codecs/ilbc/unpack_bits.c | 241 +++ .../modules/audio_coding/codecs/ilbc/unpack_bits.h | 39 + .../modules/audio_coding/codecs/ilbc/vq3.c | 64 + .../modules/audio_coding/codecs/ilbc/vq3.h | 36 + .../modules/audio_coding/codecs/ilbc/vq4.c | 63 + .../modules/audio_coding/codecs/ilbc/vq4.h | 36 + .../audio_coding/codecs/ilbc/window32_w32.c | 64 + .../audio_coding/codecs/ilbc/window32_w32.h | 35 + .../modules/audio_coding/codecs/ilbc/xcorr_coef.c | 142 ++ .../modules/audio_coding/codecs/ilbc/xcorr_coef.h | 39 + .../audio_coding/codecs/isac/bandwidth_info.h | 24 + .../codecs/isac/main/source/filter_functions.c | 195 ++ .../codecs/isac/main/source/filter_functions.h | 25 + .../codecs/isac/main/source/isac_vad.c | 409 ++++ .../codecs/isac/main/source/isac_vad.h | 45 + .../codecs/isac/main/source/os_specific_inline.h | 42 + .../codecs/isac/main/source/pitch_estimator.c | 695 +++++++ .../codecs/isac/main/source/pitch_estimator.h | 32 + .../codecs/isac/main/source/pitch_filter.c | 388 ++++ .../codecs/isac/main/source/pitch_filter.h | 42 + .../codecs/isac/main/source/settings.h | 196 ++ .../audio_coding/codecs/isac/main/source/structs.h | 448 ++++ .../codecs/legacy_encoded_audio_frame.cc | 88 + .../codecs/legacy_encoded_audio_frame.h | 53 + .../codecs/legacy_encoded_audio_frame_unittest.cc | 179 ++ .../modules/audio_coding/codecs/opus/DEPS | 5 + .../codecs/opus/audio_coder_opus_common.cc | 52 + .../codecs/opus/audio_coder_opus_common.h | 89 + .../opus/audio_decoder_multi_channel_opus_impl.cc | 182 ++ .../opus/audio_decoder_multi_channel_opus_impl.h | 74 + .../audio_decoder_multi_channel_opus_unittest.cc | 148 ++ .../audio_coding/codecs/opus/audio_decoder_opus.cc | 128 ++ .../audio_coding/codecs/opus/audio_decoder_opus.h | 64 + .../opus/audio_encoder_multi_channel_opus_impl.cc | 366 ++++ .../opus/audio_encoder_multi_channel_opus_impl.h | 92 + .../audio_encoder_multi_channel_opus_unittest.cc | 156 ++ .../audio_coding/codecs/opus/audio_encoder_opus.cc | 824 ++++++++ .../audio_coding/codecs/opus/audio_encoder_opus.h | 184 ++ .../codecs/opus/audio_encoder_opus_unittest.cc | 914 +++++++++ .../codecs/opus/opus_bandwidth_unittest.cc | 152 ++ .../codecs/opus/opus_complexity_unittest.cc | 105 + .../audio_coding/codecs/opus/opus_fec_test.cc | 248 +++ .../modules/audio_coding/codecs/opus/opus_inst.h | 43 + .../audio_coding/codecs/opus/opus_interface.cc | 881 ++++++++ .../audio_coding/codecs/opus/opus_interface.h | 547 +++++ .../audio_coding/codecs/opus/opus_speed_test.cc | 147 ++ .../audio_coding/codecs/opus/opus_unittest.cc | 979 +++++++++ .../modules/audio_coding/codecs/opus/test/BUILD.gn | 55 + .../codecs/opus/test/audio_ring_buffer.cc | 76 + .../codecs/opus/test/audio_ring_buffer.h | 57 + .../codecs/opus/test/audio_ring_buffer_unittest.cc | 111 + .../audio_coding/codecs/opus/test/blocker.cc | 215 ++ .../audio_coding/codecs/opus/test/blocker.h | 127 ++ .../codecs/opus/test/blocker_unittest.cc | 293 +++ .../codecs/opus/test/lapped_transform.cc | 100 + .../codecs/opus/test/lapped_transform.h | 175 ++ .../codecs/opus/test/lapped_transform_unittest.cc | 203 ++ .../codecs/pcm16b/audio_decoder_pcm16b.cc | 70 + .../codecs/pcm16b/audio_decoder_pcm16b.h | 52 + .../codecs/pcm16b/audio_encoder_pcm16b.cc | 39 + .../codecs/pcm16b/audio_encoder_pcm16b.h | 46 + .../modules/audio_coding/codecs/pcm16b/pcm16b.c | 32 + .../modules/audio_coding/codecs/pcm16b/pcm16b.h | 63 + .../audio_coding/codecs/pcm16b/pcm16b_common.cc | 29 + .../audio_coding/codecs/pcm16b/pcm16b_common.h | 22 + .../codecs/red/audio_encoder_copy_red.cc | 272 +++ .../codecs/red/audio_encoder_copy_red.h | 102 + .../codecs/red/audio_encoder_copy_red_unittest.cc | 641 ++++++ .../codecs/tools/audio_codec_speed_test.cc | 126 ++ .../codecs/tools/audio_codec_speed_test.h | 93 + .../default_neteq_factory_gn/moz.build | 232 +++ .../libwebrtc/modules/audio_coding/g3doc/index.md | 32 + .../modules/audio_coding/g711_c_gn/moz.build | 217 ++ .../modules/audio_coding/g711_gn/moz.build | 226 +++ .../modules/audio_coding/g722_c_gn/moz.build | 217 ++ .../modules/audio_coding/g722_gn/moz.build | 226 +++ .../modules/audio_coding/ilbc_c_gn/moz.build | 300 +++ .../modules/audio_coding/ilbc_gn/moz.build | 233 +++ .../audio_coding/include/audio_coding_module.h | 246 +++ .../include/audio_coding_module_typedefs.h | 137 ++ .../modules/audio_coding/isac_bwinfo_gn/moz.build | 201 ++ .../modules/audio_coding/isac_vad_gn/moz.build | 220 ++ .../legacy_encoded_audio_frame_gn/moz.build | 225 ++ .../modules/audio_coding/neteq/accelerate.cc | 106 + .../modules/audio_coding/neteq/accelerate.h | 79 + .../audio_coding/neteq/audio_decoder_unittest.cc | 526 +++++ .../audio_coding/neteq/audio_multi_vector.cc | 222 ++ .../audio_coding/neteq/audio_multi_vector.h | 138 ++ .../neteq/audio_multi_vector_unittest.cc | 323 +++ .../modules/audio_coding/neteq/audio_vector.cc | 381 ++++ .../modules/audio_coding/neteq/audio_vector.h | 172 ++ .../audio_coding/neteq/audio_vector_unittest.cc | 384 ++++ .../modules/audio_coding/neteq/background_noise.cc | 309 +++ .../modules/audio_coding/neteq/background_noise.h | 138 ++ .../neteq/background_noise_unittest.cc | 26 + .../audio_coding/neteq/buffer_level_filter.cc | 64 + .../audio_coding/neteq/buffer_level_filter.h | 54 + .../neteq/buffer_level_filter_unittest.cc | 116 ++ .../modules/audio_coding/neteq/comfort_noise.cc | 130 ++ .../modules/audio_coding/neteq/comfort_noise.h | 72 + .../audio_coding/neteq/comfort_noise_unittest.cc | 31 + .../audio_coding/neteq/cross_correlation.cc | 55 + .../modules/audio_coding/neteq/cross_correlation.h | 51 + .../modules/audio_coding/neteq/decision_logic.cc | 515 +++++ .../modules/audio_coding/neteq/decision_logic.h | 202 ++ .../audio_coding/neteq/decision_logic_unittest.cc | 219 ++ .../modules/audio_coding/neteq/decoder_database.cc | 285 +++ .../modules/audio_coding/neteq/decoder_database.h | 204 ++ .../neteq/decoder_database_unittest.cc | 227 +++ .../audio_coding/neteq/default_neteq_factory.cc | 31 + .../audio_coding/neteq/default_neteq_factory.h | 41 + .../modules/audio_coding/neteq/delay_manager.cc | 207 ++ .../modules/audio_coding/neteq/delay_manager.h | 121 ++ .../audio_coding/neteq/delay_manager_unittest.cc | 246 +++ .../modules/audio_coding/neteq/dsp_helper.cc | 373 ++++ .../modules/audio_coding/neteq/dsp_helper.h | 161 ++ .../audio_coding/neteq/dsp_helper_unittest.cc | 88 + .../modules/audio_coding/neteq/dtmf_buffer.cc | 246 +++ .../modules/audio_coding/neteq/dtmf_buffer.h | 104 + .../audio_coding/neteq/dtmf_buffer_unittest.cc | 297 +++ .../audio_coding/neteq/dtmf_tone_generator.cc | 215 ++ .../audio_coding/neteq/dtmf_tone_generator.h | 57 + .../neteq/dtmf_tone_generator_unittest.cc | 180 ++ .../libwebrtc/modules/audio_coding/neteq/expand.cc | 888 ++++++++ .../libwebrtc/modules/audio_coding/neteq/expand.h | 154 ++ .../audio_coding/neteq/expand_uma_logger.cc | 71 + .../modules/audio_coding/neteq/expand_uma_logger.h | 57 + .../modules/audio_coding/neteq/expand_unittest.cc | 203 ++ .../modules/audio_coding/neteq/g3doc/index.md | 102 + .../modules/audio_coding/neteq/histogram.cc | 149 ++ .../modules/audio_coding/neteq/histogram.h | 64 + .../audio_coding/neteq/histogram_unittest.cc | 73 + .../libwebrtc/modules/audio_coding/neteq/merge.cc | 391 ++++ .../libwebrtc/modules/audio_coding/neteq/merge.h | 101 + .../modules/audio_coding/neteq/merge_unittest.cc | 121 ++ .../neteq/mock/mock_buffer_level_filter.h | 28 + .../neteq/mock/mock_decoder_database.h | 50 + .../audio_coding/neteq/mock/mock_delay_manager.h | 29 + .../audio_coding/neteq/mock/mock_dtmf_buffer.h | 35 + .../neteq/mock/mock_dtmf_tone_generator.h | 33 + .../modules/audio_coding/neteq/mock/mock_expand.h | 60 + .../audio_coding/neteq/mock/mock_histogram.h | 30 + .../neteq/mock/mock_neteq_controller.h | 62 + .../audio_coding/neteq/mock/mock_packet_buffer.h | 82 + .../neteq/mock/mock_red_payload_splitter.h | 30 + .../neteq/mock/mock_statistics_calculator.h | 30 + .../modules/audio_coding/neteq/nack_tracker.cc | 267 +++ .../modules/audio_coding/neteq/nack_tracker.h | 211 ++ .../audio_coding/neteq/nack_tracker_unittest.cc | 565 ++++++ .../neteq/neteq_decoder_plc_unittest.cc | 313 +++ .../modules/audio_coding/neteq/neteq_impl.cc | 2141 ++++++++++++++++++++ .../modules/audio_coding/neteq/neteq_impl.h | 404 ++++ .../audio_coding/neteq/neteq_impl_unittest.cc | 1871 +++++++++++++++++ .../neteq/neteq_network_stats_unittest.cc | 345 ++++ .../audio_coding/neteq/neteq_stereo_unittest.cc | 424 ++++ .../modules/audio_coding/neteq/neteq_unittest.cc | 1009 +++++++++ .../audio_coding/neteq/neteq_unittest.proto | 31 + .../libwebrtc/modules/audio_coding/neteq/normal.cc | 194 ++ .../libwebrtc/modules/audio_coding/neteq/normal.h | 76 + .../modules/audio_coding/neteq/normal_unittest.cc | 147 ++ .../libwebrtc/modules/audio_coding/neteq/packet.cc | 36 + .../libwebrtc/modules/audio_coding/neteq/packet.h | 128 ++ .../audio_coding/neteq/packet_arrival_history.cc | 105 + .../audio_coding/neteq/packet_arrival_history.h | 82 + .../neteq/packet_arrival_history_unittest.cc | 141 ++ .../modules/audio_coding/neteq/packet_buffer.cc | 405 ++++ .../modules/audio_coding/neteq/packet_buffer.h | 181 ++ .../audio_coding/neteq/packet_buffer_unittest.cc | 989 +++++++++ .../modules/audio_coding/neteq/post_decode_vad.cc | 90 + .../modules/audio_coding/neteq/post_decode_vad.h | 71 + .../audio_coding/neteq/post_decode_vad_unittest.cc | 25 + .../audio_coding/neteq/preemptive_expand.cc | 117 ++ .../modules/audio_coding/neteq/preemptive_expand.h | 85 + .../modules/audio_coding/neteq/random_vector.cc | 63 + .../modules/audio_coding/neteq/random_vector.h | 46 + .../audio_coding/neteq/random_vector_unittest.cc | 25 + .../audio_coding/neteq/red_payload_splitter.cc | 190 ++ .../audio_coding/neteq/red_payload_splitter.h | 51 + .../neteq/red_payload_splitter_unittest.cc | 390 ++++ .../audio_coding/neteq/reorder_optimizer.cc | 75 + .../modules/audio_coding/neteq/reorder_optimizer.h | 43 + .../neteq/reorder_optimizer_unittest.cc | 70 + .../audio_coding/neteq/statistics_calculator.cc | 394 ++++ .../audio_coding/neteq/statistics_calculator.h | 210 ++ .../neteq/statistics_calculator_unittest.cc | 206 ++ .../modules/audio_coding/neteq/sync_buffer.cc | 118 ++ .../modules/audio_coding/neteq/sync_buffer.h | 110 + .../audio_coding/neteq/sync_buffer_unittest.cc | 174 ++ .../neteq/test/delay_tool/parse_delay_file.m | 201 ++ .../neteq/test/delay_tool/plot_neteq_delay.m | 197 ++ .../audio_coding/neteq/test/neteq_decoding_test.cc | 423 ++++ .../audio_coding/neteq/test/neteq_decoding_test.h | 96 + .../neteq/test/neteq_ilbc_quality_test.cc | 81 + .../neteq/test/neteq_opus_quality_test.cc | 183 ++ .../neteq/test/neteq_pcm16b_quality_test.cc | 81 + .../neteq/test/neteq_pcmu_quality_test.cc | 80 + .../neteq/test/neteq_performance_unittest.cc | 60 + .../audio_coding/neteq/test/neteq_speed_test.cc | 58 + .../modules/audio_coding/neteq/test/result_sink.cc | 109 + .../modules/audio_coding/neteq/test/result_sink.h | 50 + .../modules/audio_coding/neteq/time_stretch.cc | 216 ++ .../modules/audio_coding/neteq/time_stretch.h | 113 ++ .../audio_coding/neteq/time_stretch_unittest.cc | 124 ++ .../modules/audio_coding/neteq/timestamp_scaler.cc | 87 + .../modules/audio_coding/neteq/timestamp_scaler.h | 67 + .../neteq/timestamp_scaler_unittest.cc | 324 +++ .../modules/audio_coding/neteq/tools/DEPS | 3 + .../modules/audio_coding/neteq/tools/README.md | 17 + .../audio_coding/neteq/tools/audio_checksum.h | 64 + .../modules/audio_coding/neteq/tools/audio_loop.cc | 61 + .../modules/audio_coding/neteq/tools/audio_loop.h | 57 + .../modules/audio_coding/neteq/tools/audio_sink.cc | 26 + .../modules/audio_coding/neteq/tools/audio_sink.h | 70 + .../neteq/tools/constant_pcm_packet_source.cc | 71 + .../neteq/tools/constant_pcm_packet_source.h | 55 + .../audio_coding/neteq/tools/encode_neteq_input.cc | 94 + .../audio_coding/neteq/tools/encode_neteq_input.h | 69 + .../neteq/tools/fake_decode_from_file.cc | 169 ++ .../neteq/tools/fake_decode_from_file.h | 77 + .../tools/initial_packet_inserter_neteq_input.cc | 79 + .../tools/initial_packet_inserter_neteq_input.h | 46 + .../audio_coding/neteq/tools/input_audio_file.cc | 96 + .../audio_coding/neteq/tools/input_audio_file.h | 62 + .../neteq/tools/input_audio_file_unittest.cc | 59 + .../neteq/tools/neteq_delay_analyzer.cc | 307 +++ .../neteq/tools/neteq_delay_analyzer.h | 76 + .../neteq/tools/neteq_event_log_input.cc | 68 + .../neteq/tools/neteq_event_log_input.h | 51 + .../audio_coding/neteq/tools/neteq_input.cc | 93 + .../modules/audio_coding/neteq/tools/neteq_input.h | 107 + .../neteq/tools/neteq_packet_source_input.cc | 90 + .../neteq/tools/neteq_packet_source_input.h | 70 + .../neteq/tools/neteq_performance_test.cc | 128 ++ .../neteq/tools/neteq_performance_test.h | 32 + .../audio_coding/neteq/tools/neteq_quality_test.cc | 482 +++++ .../audio_coding/neteq/tools/neteq_quality_test.h | 176 ++ .../neteq/tools/neteq_replacement_input.cc | 116 ++ .../neteq/tools/neteq_replacement_input.h | 51 + .../audio_coding/neteq/tools/neteq_rtpplay.cc | 406 ++++ .../audio_coding/neteq/tools/neteq_rtpplay_test.sh | 183 ++ .../audio_coding/neteq/tools/neteq_stats_getter.cc | 139 ++ .../audio_coding/neteq/tools/neteq_stats_getter.h | 106 + .../neteq/tools/neteq_stats_plotter.cc | 111 + .../audio_coding/neteq/tools/neteq_stats_plotter.h | 47 + .../modules/audio_coding/neteq/tools/neteq_test.cc | 345 ++++ .../modules/audio_coding/neteq/tools/neteq_test.h | 129 ++ .../audio_coding/neteq/tools/neteq_test_factory.cc | 342 ++++ .../audio_coding/neteq/tools/neteq_test_factory.h | 172 ++ .../audio_coding/neteq/tools/output_audio_file.h | 51 + .../audio_coding/neteq/tools/output_wav_file.h | 46 + .../modules/audio_coding/neteq/tools/packet.cc | 133 ++ .../modules/audio_coding/neteq/tools/packet.h | 104 + .../audio_coding/neteq/tools/packet_source.cc | 25 + .../audio_coding/neteq/tools/packet_source.h | 43 + .../audio_coding/neteq/tools/packet_unittest.cc | 226 +++ .../neteq/tools/resample_input_audio_file.cc | 48 + .../neteq/tools/resample_input_audio_file.h | 55 + .../neteq/tools/rtc_event_log_source.cc | 168 ++ .../neteq/tools/rtc_event_log_source.h | 71 + .../audio_coding/neteq/tools/rtp_analyze.cc | 166 ++ .../modules/audio_coding/neteq/tools/rtp_encode.cc | 351 ++++ .../audio_coding/neteq/tools/rtp_file_source.cc | 100 + .../audio_coding/neteq/tools/rtp_file_source.h | 68 + .../audio_coding/neteq/tools/rtp_generator.cc | 60 + .../audio_coding/neteq/tools/rtp_generator.h | 83 + .../modules/audio_coding/neteq/tools/rtp_jitter.cc | 148 ++ .../modules/audio_coding/neteq/tools/rtpcat.cc | 45 + .../audio_coding/neteq/underrun_optimizer.cc | 71 + .../audio_coding/neteq/underrun_optimizer.h | 50 + .../neteq/underrun_optimizer_unittest.cc | 42 + .../modules/audio_coding/neteq_gn/moz.build | 267 +++ .../modules/audio_coding/pcm16b_c_gn/moz.build | 217 ++ .../modules/audio_coding/pcm16b_gn/moz.build | 227 +++ .../modules/audio_coding/red_gn/moz.build | 232 +++ .../libwebrtc/modules/audio_coding/test/Channel.cc | 274 +++ .../libwebrtc/modules/audio_coding/test/Channel.h | 117 ++ .../modules/audio_coding/test/EncodeDecodeTest.cc | 269 +++ .../modules/audio_coding/test/EncodeDecodeTest.h | 111 + .../libwebrtc/modules/audio_coding/test/PCMFile.cc | 240 +++ .../libwebrtc/modules/audio_coding/test/PCMFile.h | 77 + .../modules/audio_coding/test/PacketLossTest.cc | 167 ++ .../modules/audio_coding/test/PacketLossTest.h | 77 + .../libwebrtc/modules/audio_coding/test/RTPFile.cc | 235 +++ .../libwebrtc/modules/audio_coding/test/RTPFile.h | 133 ++ .../modules/audio_coding/test/TestAllCodecs.cc | 412 ++++ .../modules/audio_coding/test/TestAllCodecs.h | 83 + .../modules/audio_coding/test/TestRedFec.cc | 200 ++ .../modules/audio_coding/test/TestRedFec.h | 56 + .../modules/audio_coding/test/TestStereo.cc | 599 ++++++ .../modules/audio_coding/test/TestStereo.h | 100 + .../modules/audio_coding/test/TestVADDTX.cc | 240 +++ .../modules/audio_coding/test/TestVADDTX.h | 115 ++ .../libwebrtc/modules/audio_coding/test/Tester.cc | 102 + .../audio_coding/test/TwoWayCommunication.cc | 191 ++ .../audio_coding/test/TwoWayCommunication.h | 62 + .../modules/audio_coding/test/opus_test.cc | 402 ++++ .../modules/audio_coding/test/opus_test.h | 59 + .../audio_coding/test/target_delay_unittest.cc | 161 ++ .../modules/audio_coding/webrtc_cng_gn/moz.build | 232 +++ .../audio_coding/webrtc_multiopus_gn/moz.build | 230 +++ .../modules/audio_coding/webrtc_opus_gn/moz.build | 237 +++ .../audio_coding/webrtc_opus_wrapper_gn/moz.build | 229 +++ 535 files changed, 83708 insertions(+) create mode 100644 third_party/libwebrtc/modules/audio_coding/BUILD.gn create mode 100644 third_party/libwebrtc/modules/audio_coding/DEPS create mode 100644 third_party/libwebrtc/modules/audio_coding/OWNERS create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_receive_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_receive_test.h create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.h create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.h create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.h create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_send_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/acm_send_test.h create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.h create mode 100644 third_party/libwebrtc/modules/audio_coding/acm2/call_statistics_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_coding.gni create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_coding_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_coding_module_typedefs_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_coding_opus_common_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_encoder_cng_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/config.proto create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h create mode 100755 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/parse_ana_dump.py create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve.h create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor_config_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/audio_network_adaptor_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/audio_decoder.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/audio_encoder.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/cng/cng_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g711/test/testG711.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/g722/test/testG722.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/complexityMeasures.m create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/defines.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/energy_inverse.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enh_upsample.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enh_upsample.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/my_corr.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/my_corr.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_construct.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_construct.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/empty.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_test.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testLib.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testprogram.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/unpack_bits.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/unpack_bits.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/bandwidth_info.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/settings.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/structs.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/DEPS create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_bandwidth_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_inst.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_speed_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/BUILD.gn create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.h create mode 100644 third_party/libwebrtc/modules/audio_coding/default_neteq_factory_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/g3doc/index.md create mode 100644 third_party/libwebrtc/modules/audio_coding/g711_c_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/g711_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/g722_c_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/g722_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/ilbc_c_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/ilbc_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/include/audio_coding_module.h create mode 100644 third_party/libwebrtc/modules/audio_coding/include/audio_coding_module_typedefs.h create mode 100644 third_party/libwebrtc/modules/audio_coding/isac_bwinfo_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/isac_vad_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/legacy_encoded_audio_frame_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/accelerate.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/accelerate.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/audio_vector_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/background_noise.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/background_noise.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/background_noise_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/decision_logic_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/decoder_database_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/delay_manager_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/expand.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/expand.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/expand_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/g3doc/index.md create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/histogram.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/histogram.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/histogram_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/merge.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/merge.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/merge_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_delay_manager.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_expand.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_histogram.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_neteq_controller.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/neteq_stereo_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/neteq_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/neteq_unittest.proto create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/normal.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/normal.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/normal_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/packet.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/packet.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/random_vector.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/random_vector.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/random_vector_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/delay_tool/parse_delay_file.m create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/delay_tool/plot_neteq_delay.m create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_decoding_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_decoding_test.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_ilbc_quality_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_pcm16b_quality_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_pcmu_quality_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_performance_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/result_sink.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/test/result_sink.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/time_stretch_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/DEPS create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/README.md create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_checksum.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_loop.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_loop.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_sink.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_sink.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/encode_neteq_input.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/encode_neteq_input.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_event_log_input.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_event_log_input.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_input.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_input.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_performance_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_performance_test.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_quality_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_quality_test.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc create mode 100755 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_rtpplay_test.sh create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_getter.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_getter.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test_factory.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test_factory.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/output_audio_file.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/output_wav_file.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/packet.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/packet.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_source.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_source.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_analyze.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_encode.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_file_source.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_file_source.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_generator.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_generator.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_jitter.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/tools/rtpcat.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.h create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/neteq_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/pcm16b_c_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/pcm16b_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/red_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/test/Channel.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/Channel.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/EncodeDecodeTest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/EncodeDecodeTest.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/PCMFile.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/PCMFile.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/PacketLossTest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/PacketLossTest.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/RTPFile.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/RTPFile.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TestAllCodecs.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TestAllCodecs.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TestRedFec.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TestRedFec.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TestStereo.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TestStereo.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TestVADDTX.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TestVADDTX.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/Tester.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TwoWayCommunication.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/TwoWayCommunication.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/opus_test.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/test/opus_test.h create mode 100644 third_party/libwebrtc/modules/audio_coding/test/target_delay_unittest.cc create mode 100644 third_party/libwebrtc/modules/audio_coding/webrtc_cng_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/webrtc_multiopus_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/webrtc_opus_gn/moz.build create mode 100644 third_party/libwebrtc/modules/audio_coding/webrtc_opus_wrapper_gn/moz.build (limited to 'third_party/libwebrtc/modules/audio_coding') diff --git a/third_party/libwebrtc/modules/audio_coding/BUILD.gn b/third_party/libwebrtc/modules/audio_coding/BUILD.gn new file mode 100644 index 0000000000..eac0650a26 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/BUILD.gn @@ -0,0 +1,1805 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") +import("audio_coding.gni") +if (rtc_enable_protobuf) { + import("//third_party/protobuf/proto_library.gni") +} + +visibility = [ ":*" ] + +rtc_source_set("audio_coding_module_typedefs") { + visibility += [ "*" ] + sources = [ "include/audio_coding_module_typedefs.h" ] +} + +rtc_library("audio_coding") { + visibility += [ "*" ] + sources = [ + "acm2/acm_receiver.cc", + "acm2/acm_receiver.h", + "acm2/acm_remixing.cc", + "acm2/acm_remixing.h", + "acm2/acm_resampler.cc", + "acm2/acm_resampler.h", + "acm2/audio_coding_module.cc", + "acm2/call_statistics.cc", + "acm2/call_statistics.h", + "include/audio_coding_module.h", + ] + + defines = [] + + deps = [ + ":audio_coding_module_typedefs", + ":default_neteq_factory", + ":neteq", + "..:module_api", + "..:module_api_public", + "../../api:array_view", + "../../api:function_view", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/neteq:neteq_api", + "../../common_audio", + "../../common_audio:common_audio_c", + "../../rtc_base:audio_format_to_string", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:safe_conversions", + "../../rtc_base/synchronization:mutex", + "../../system_wrappers", + "../../system_wrappers:metrics", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("legacy_encoded_audio_frame") { + sources = [ + "codecs/legacy_encoded_audio_frame.cc", + "codecs/legacy_encoded_audio_frame.h", + ] + deps = [ + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:buffer", + "../../rtc_base:checks", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("webrtc_cng") { + visibility += webrtc_default_visibility + sources = [ + "codecs/cng/webrtc_cng.cc", + "codecs/cng/webrtc_cng.h", + ] + + deps = [ + "../../api:array_view", + "../../common_audio:common_audio_c", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + ] +} + +rtc_library("audio_encoder_cng") { + visibility += [ "*" ] + sources = [ + "codecs/cng/audio_encoder_cng.cc", + "codecs/cng/audio_encoder_cng.h", + ] + + deps = [ + ":webrtc_cng", + "../../api/audio_codecs:audio_codecs_api", + "../../api/units:time_delta", + "../../common_audio", + "../../rtc_base:checks", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("red") { + visibility += [ "*" ] + sources = [ + "codecs/red/audio_encoder_copy_red.cc", + "codecs/red/audio_encoder_copy_red.h", + ] + + deps = [ + "../../api:array_view", + "../../api:field_trials_view", + "../../api/audio_codecs:audio_codecs_api", + "../../api/units:time_delta", + "../../common_audio", + "../../rtc_base:buffer", + "../../rtc_base:byte_order", + "../../rtc_base:checks", + "../../rtc_base:logging", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("g711") { + visibility += [ "*" ] + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/g711/audio_decoder_pcm.cc", + "codecs/g711/audio_decoder_pcm.h", + "codecs/g711/audio_encoder_pcm.cc", + "codecs/g711/audio_encoder_pcm.h", + ] + + deps = [ + ":legacy_encoded_audio_frame", + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../api/units:time_delta", + "../../rtc_base:buffer", + "../../rtc_base:checks", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + public_deps = [ ":g711_c" ] # no-presubmit-check TODO(webrtc:8603) +} + +rtc_library("g711_c") { + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/g711/g711_interface.c", + "codecs/g711/g711_interface.h", + ] + deps = [ "../third_party/g711:g711_3p" ] +} + +rtc_library("g722") { + visibility += [ "*" ] + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/g722/audio_decoder_g722.cc", + "codecs/g722/audio_decoder_g722.h", + "codecs/g722/audio_encoder_g722.cc", + "codecs/g722/audio_encoder_g722.h", + ] + + deps = [ + ":legacy_encoded_audio_frame", + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/g722:audio_encoder_g722_config", + "../../api/units:time_delta", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + public_deps = [ ":g722_c" ] # no-presubmit-check TODO(webrtc:8603) +} + +rtc_library("g722_c") { + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/g722/g722_interface.c", + "codecs/g722/g722_interface.h", + ] + deps = [ "../third_party/g722:g722_3p" ] +} + +rtc_library("ilbc") { + visibility += webrtc_default_visibility + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/ilbc/audio_decoder_ilbc.cc", + "codecs/ilbc/audio_decoder_ilbc.h", + "codecs/ilbc/audio_encoder_ilbc.cc", + "codecs/ilbc/audio_encoder_ilbc.h", + ] + + deps = [ + ":legacy_encoded_audio_frame", + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/ilbc:audio_encoder_ilbc_config", + "../../api/units:time_delta", + "../../common_audio", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:safe_conversions", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + public_deps = [ ":ilbc_c" ] # no-presubmit-check TODO(webrtc:8603) +} + +rtc_library("ilbc_c") { + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/ilbc/abs_quant.c", + "codecs/ilbc/abs_quant.h", + "codecs/ilbc/abs_quant_loop.c", + "codecs/ilbc/abs_quant_loop.h", + "codecs/ilbc/augmented_cb_corr.c", + "codecs/ilbc/augmented_cb_corr.h", + "codecs/ilbc/bw_expand.c", + "codecs/ilbc/bw_expand.h", + "codecs/ilbc/cb_construct.c", + "codecs/ilbc/cb_construct.h", + "codecs/ilbc/cb_mem_energy.c", + "codecs/ilbc/cb_mem_energy.h", + "codecs/ilbc/cb_mem_energy_augmentation.c", + "codecs/ilbc/cb_mem_energy_augmentation.h", + "codecs/ilbc/cb_mem_energy_calc.c", + "codecs/ilbc/cb_mem_energy_calc.h", + "codecs/ilbc/cb_search.c", + "codecs/ilbc/cb_search.h", + "codecs/ilbc/cb_search_core.c", + "codecs/ilbc/cb_search_core.h", + "codecs/ilbc/cb_update_best_index.c", + "codecs/ilbc/cb_update_best_index.h", + "codecs/ilbc/chebyshev.c", + "codecs/ilbc/chebyshev.h", + "codecs/ilbc/comp_corr.c", + "codecs/ilbc/comp_corr.h", + "codecs/ilbc/constants.c", + "codecs/ilbc/constants.h", + "codecs/ilbc/create_augmented_vec.c", + "codecs/ilbc/create_augmented_vec.h", + "codecs/ilbc/decode.c", + "codecs/ilbc/decode.h", + "codecs/ilbc/decode_residual.c", + "codecs/ilbc/decode_residual.h", + "codecs/ilbc/decoder_interpolate_lsf.c", + "codecs/ilbc/decoder_interpolate_lsf.h", + "codecs/ilbc/defines.h", + "codecs/ilbc/do_plc.c", + "codecs/ilbc/do_plc.h", + "codecs/ilbc/encode.c", + "codecs/ilbc/encode.h", + "codecs/ilbc/energy_inverse.c", + "codecs/ilbc/energy_inverse.h", + "codecs/ilbc/enh_upsample.c", + "codecs/ilbc/enh_upsample.h", + "codecs/ilbc/enhancer.c", + "codecs/ilbc/enhancer.h", + "codecs/ilbc/enhancer_interface.c", + "codecs/ilbc/enhancer_interface.h", + "codecs/ilbc/filtered_cb_vecs.c", + "codecs/ilbc/filtered_cb_vecs.h", + "codecs/ilbc/frame_classify.c", + "codecs/ilbc/frame_classify.h", + "codecs/ilbc/gain_dequant.c", + "codecs/ilbc/gain_dequant.h", + "codecs/ilbc/gain_quant.c", + "codecs/ilbc/gain_quant.h", + "codecs/ilbc/get_cd_vec.c", + "codecs/ilbc/get_cd_vec.h", + "codecs/ilbc/get_lsp_poly.c", + "codecs/ilbc/get_lsp_poly.h", + "codecs/ilbc/get_sync_seq.c", + "codecs/ilbc/get_sync_seq.h", + "codecs/ilbc/hp_input.c", + "codecs/ilbc/hp_input.h", + "codecs/ilbc/hp_output.c", + "codecs/ilbc/hp_output.h", + "codecs/ilbc/ilbc.c", + "codecs/ilbc/ilbc.h", + "codecs/ilbc/index_conv_dec.c", + "codecs/ilbc/index_conv_dec.h", + "codecs/ilbc/index_conv_enc.c", + "codecs/ilbc/index_conv_enc.h", + "codecs/ilbc/init_decode.c", + "codecs/ilbc/init_decode.h", + "codecs/ilbc/init_encode.c", + "codecs/ilbc/init_encode.h", + "codecs/ilbc/interpolate.c", + "codecs/ilbc/interpolate.h", + "codecs/ilbc/interpolate_samples.c", + "codecs/ilbc/interpolate_samples.h", + "codecs/ilbc/lpc_encode.c", + "codecs/ilbc/lpc_encode.h", + "codecs/ilbc/lsf_check.c", + "codecs/ilbc/lsf_check.h", + "codecs/ilbc/lsf_interpolate_to_poly_dec.c", + "codecs/ilbc/lsf_interpolate_to_poly_dec.h", + "codecs/ilbc/lsf_interpolate_to_poly_enc.c", + "codecs/ilbc/lsf_interpolate_to_poly_enc.h", + "codecs/ilbc/lsf_to_lsp.c", + "codecs/ilbc/lsf_to_lsp.h", + "codecs/ilbc/lsf_to_poly.c", + "codecs/ilbc/lsf_to_poly.h", + "codecs/ilbc/lsp_to_lsf.c", + "codecs/ilbc/lsp_to_lsf.h", + "codecs/ilbc/my_corr.c", + "codecs/ilbc/my_corr.h", + "codecs/ilbc/nearest_neighbor.c", + "codecs/ilbc/nearest_neighbor.h", + "codecs/ilbc/pack_bits.c", + "codecs/ilbc/pack_bits.h", + "codecs/ilbc/poly_to_lsf.c", + "codecs/ilbc/poly_to_lsf.h", + "codecs/ilbc/poly_to_lsp.c", + "codecs/ilbc/poly_to_lsp.h", + "codecs/ilbc/refiner.c", + "codecs/ilbc/refiner.h", + "codecs/ilbc/simple_interpolate_lsf.c", + "codecs/ilbc/simple_interpolate_lsf.h", + "codecs/ilbc/simple_lpc_analysis.c", + "codecs/ilbc/simple_lpc_analysis.h", + "codecs/ilbc/simple_lsf_dequant.c", + "codecs/ilbc/simple_lsf_dequant.h", + "codecs/ilbc/simple_lsf_quant.c", + "codecs/ilbc/simple_lsf_quant.h", + "codecs/ilbc/smooth.c", + "codecs/ilbc/smooth.h", + "codecs/ilbc/smooth_out_data.c", + "codecs/ilbc/smooth_out_data.h", + "codecs/ilbc/sort_sq.c", + "codecs/ilbc/sort_sq.h", + "codecs/ilbc/split_vq.c", + "codecs/ilbc/split_vq.h", + "codecs/ilbc/state_construct.c", + "codecs/ilbc/state_construct.h", + "codecs/ilbc/state_search.c", + "codecs/ilbc/state_search.h", + "codecs/ilbc/swap_bytes.c", + "codecs/ilbc/swap_bytes.h", + "codecs/ilbc/unpack_bits.c", + "codecs/ilbc/unpack_bits.h", + "codecs/ilbc/vq3.c", + "codecs/ilbc/vq3.h", + "codecs/ilbc/vq4.c", + "codecs/ilbc/vq4.h", + "codecs/ilbc/window32_w32.c", + "codecs/ilbc/window32_w32.h", + "codecs/ilbc/xcorr_coef.c", + "codecs/ilbc/xcorr_coef.h", + ] + + deps = [ + "../../api/audio_codecs:audio_codecs_api", + "../../common_audio", + "../../common_audio:common_audio_c", + "../../rtc_base:checks", + "../../rtc_base:sanitizer", + "../../rtc_base/system:arch", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] +} + +rtc_library("isac_vad") { + visibility += [ "../audio_processing/vad:*" ] + sources = [ + "codecs/isac/main/source/filter_functions.c", + "codecs/isac/main/source/filter_functions.h", + "codecs/isac/main/source/isac_vad.c", + "codecs/isac/main/source/isac_vad.h", + "codecs/isac/main/source/os_specific_inline.h", + "codecs/isac/main/source/pitch_estimator.c", + "codecs/isac/main/source/pitch_estimator.h", + "codecs/isac/main/source/pitch_filter.c", + "codecs/isac/main/source/pitch_filter.h", + "codecs/isac/main/source/settings.h", + "codecs/isac/main/source/structs.h", + ] + deps = [ + ":isac_bwinfo", + "../../rtc_base:compile_assert_c", + "../../rtc_base/system:arch", + "../../rtc_base/system:ignore_warnings", + "../third_party/fft", + ] +} + +rtc_source_set("isac_bwinfo") { + sources = [ "codecs/isac/bandwidth_info.h" ] + deps = [] +} + +rtc_library("pcm16b") { + visibility += [ "*" ] + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/pcm16b/audio_decoder_pcm16b.cc", + "codecs/pcm16b/audio_decoder_pcm16b.h", + "codecs/pcm16b/audio_encoder_pcm16b.cc", + "codecs/pcm16b/audio_encoder_pcm16b.h", + "codecs/pcm16b/pcm16b_common.cc", + "codecs/pcm16b/pcm16b_common.h", + ] + + deps = [ + ":g711", + ":legacy_encoded_audio_frame", + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:buffer", + "../../rtc_base:checks", + ] + public_deps = [ ":pcm16b_c" ] # no-presubmit-check TODO(webrtc:8603) +} + +rtc_library("pcm16b_c") { + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/pcm16b/pcm16b.c", + "codecs/pcm16b/pcm16b.h", + ] +} + +rtc_library("audio_coding_opus_common") { + sources = [ + "codecs/opus/audio_coder_opus_common.cc", + "codecs/opus/audio_coder_opus_common.h", + ] + + deps = [ + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:checks", + "../../rtc_base:stringutils", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("webrtc_opus") { + visibility += webrtc_default_visibility + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/opus/audio_decoder_opus.cc", + "codecs/opus/audio_decoder_opus.h", + "codecs/opus/audio_encoder_opus.cc", + "codecs/opus/audio_encoder_opus.h", + ] + + deps = [ + ":audio_coding_opus_common", + ":audio_network_adaptor", + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/opus:audio_encoder_opus_config", + "../../common_audio", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:protobuf_utils", + "../../rtc_base:rtc_numerics", + "../../rtc_base:safe_conversions", + "../../rtc_base:safe_minmax", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", + "../../system_wrappers:field_trial", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + public_deps = # no-presubmit-check TODO(webrtc:8603) + [ ":webrtc_opus_wrapper" ] + + defines = audio_codec_defines +} + +rtc_library("webrtc_multiopus") { + visibility += webrtc_default_visibility + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/opus/audio_decoder_multi_channel_opus_impl.cc", + "codecs/opus/audio_decoder_multi_channel_opus_impl.h", + "codecs/opus/audio_encoder_multi_channel_opus_impl.cc", + "codecs/opus/audio_encoder_multi_channel_opus_impl.h", + ] + + deps = [ + ":audio_coding_opus_common", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/opus:audio_decoder_opus_config", + "../../api/audio_codecs/opus:audio_encoder_opus_config", + "../../api/units:time_delta", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:safe_minmax", + "../../rtc_base:stringutils", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + public_deps = # no-presubmit-check TODO(webrtc:8603) + [ ":webrtc_opus_wrapper" ] + + defines = audio_codec_defines +} + +rtc_library("webrtc_opus_wrapper") { + poisonous = [ "audio_codecs" ] + sources = [ + "codecs/opus/opus_inst.h", + "codecs/opus/opus_interface.cc", + "codecs/opus/opus_interface.h", + ] + + defines = audio_coding_defines + + deps = [ + "../../api:array_view", + "../../rtc_base:checks", + "../../rtc_base:ignore_wundef", + "../../system_wrappers:field_trial", + ] + + if (rtc_build_opus) { + deps += [ rtc_opus_dir ] + public_configs = [ "//third_party/opus:opus_config" ] + } else if (build_with_mozilla) { + public_configs = [ "//third_party/opus:opus_config" ] + } +} + +if (rtc_enable_protobuf) { + proto_library("ana_debug_dump_proto") { + visibility += webrtc_default_visibility + sources = [ "audio_network_adaptor/debug_dump.proto" ] + link_deps = [ ":ana_config_proto" ] + proto_out_dir = "modules/audio_coding/audio_network_adaptor" + } + proto_library("ana_config_proto") { + visibility += [ "*" ] + sources = [ "audio_network_adaptor/config.proto" ] + proto_out_dir = "modules/audio_coding/audio_network_adaptor" + } +} + +rtc_library("audio_network_adaptor_config") { + visibility += webrtc_default_visibility + sources = [ + "audio_network_adaptor/audio_network_adaptor_config.cc", + "audio_network_adaptor/include/audio_network_adaptor_config.h", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("audio_network_adaptor") { + visibility += webrtc_default_visibility + sources = [ + "audio_network_adaptor/audio_network_adaptor_impl.cc", + "audio_network_adaptor/audio_network_adaptor_impl.h", + "audio_network_adaptor/bitrate_controller.cc", + "audio_network_adaptor/bitrate_controller.h", + "audio_network_adaptor/channel_controller.cc", + "audio_network_adaptor/channel_controller.h", + "audio_network_adaptor/controller.cc", + "audio_network_adaptor/controller.h", + "audio_network_adaptor/controller_manager.cc", + "audio_network_adaptor/controller_manager.h", + "audio_network_adaptor/debug_dump_writer.cc", + "audio_network_adaptor/debug_dump_writer.h", + "audio_network_adaptor/dtx_controller.cc", + "audio_network_adaptor/dtx_controller.h", + "audio_network_adaptor/event_log_writer.cc", + "audio_network_adaptor/event_log_writer.h", + "audio_network_adaptor/fec_controller_plr_based.cc", + "audio_network_adaptor/fec_controller_plr_based.h", + "audio_network_adaptor/frame_length_controller.cc", + "audio_network_adaptor/frame_length_controller.h", + "audio_network_adaptor/frame_length_controller_v2.cc", + "audio_network_adaptor/frame_length_controller_v2.h", + "audio_network_adaptor/include/audio_network_adaptor.h", + "audio_network_adaptor/util/threshold_curve.h", + ] + + public_deps = # no-presubmit-check TODO(webrtc:8603) + [ ":audio_network_adaptor_config" ] + + deps = [ + "../../api/audio_codecs:audio_codecs_api", + "../../api/rtc_event_log", + "../../common_audio", + "../../logging:rtc_event_audio", + "../../rtc_base:checks", + "../../rtc_base:ignore_wundef", + "../../rtc_base:logging", + "../../rtc_base:protobuf_utils", + "../../rtc_base:safe_conversions", + "../../rtc_base:timeutils", + "../../rtc_base/system:file_wrapper", + "../../system_wrappers", + "../../system_wrappers:field_trial", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + + if (rtc_enable_protobuf) { + deps += [ + ":ana_config_proto", + ":ana_debug_dump_proto", + ] + } +} + +rtc_library("neteq") { + visibility += webrtc_default_visibility + sources = [ + "neteq/accelerate.cc", + "neteq/accelerate.h", + "neteq/audio_multi_vector.cc", + "neteq/audio_multi_vector.h", + "neteq/audio_vector.cc", + "neteq/audio_vector.h", + "neteq/background_noise.cc", + "neteq/background_noise.h", + "neteq/buffer_level_filter.cc", + "neteq/buffer_level_filter.h", + "neteq/comfort_noise.cc", + "neteq/comfort_noise.h", + "neteq/cross_correlation.cc", + "neteq/cross_correlation.h", + "neteq/decision_logic.cc", + "neteq/decision_logic.h", + "neteq/decoder_database.cc", + "neteq/decoder_database.h", + "neteq/delay_manager.cc", + "neteq/delay_manager.h", + "neteq/dsp_helper.cc", + "neteq/dsp_helper.h", + "neteq/dtmf_buffer.cc", + "neteq/dtmf_buffer.h", + "neteq/dtmf_tone_generator.cc", + "neteq/dtmf_tone_generator.h", + "neteq/expand.cc", + "neteq/expand.h", + "neteq/expand_uma_logger.cc", + "neteq/expand_uma_logger.h", + "neteq/histogram.cc", + "neteq/histogram.h", + "neteq/merge.cc", + "neteq/merge.h", + "neteq/nack_tracker.cc", + "neteq/nack_tracker.h", + "neteq/neteq_impl.cc", + "neteq/neteq_impl.h", + "neteq/normal.cc", + "neteq/normal.h", + "neteq/packet.cc", + "neteq/packet.h", + "neteq/packet_arrival_history.cc", + "neteq/packet_arrival_history.h", + "neteq/packet_buffer.cc", + "neteq/packet_buffer.h", + "neteq/post_decode_vad.cc", + "neteq/post_decode_vad.h", + "neteq/preemptive_expand.cc", + "neteq/preemptive_expand.h", + "neteq/random_vector.cc", + "neteq/random_vector.h", + "neteq/red_payload_splitter.cc", + "neteq/red_payload_splitter.h", + "neteq/reorder_optimizer.cc", + "neteq/reorder_optimizer.h", + "neteq/statistics_calculator.cc", + "neteq/statistics_calculator.h", + "neteq/sync_buffer.cc", + "neteq/sync_buffer.h", + "neteq/time_stretch.cc", + "neteq/time_stretch.h", + "neteq/timestamp_scaler.cc", + "neteq/timestamp_scaler.h", + "neteq/underrun_optimizer.cc", + "neteq/underrun_optimizer.h", + ] + + deps = [ + ":audio_coding_module_typedefs", + ":webrtc_cng", + "..:module_api_public", + "../../api:array_view", + "../../api:rtp_headers", + "../../api:rtp_packet_info", + "../../api:scoped_refptr", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/neteq:neteq_api", + "../../api/neteq:neteq_controller_api", + "../../api/neteq:tick_timer", + "../../common_audio", + "../../common_audio:common_audio_c", + "../../rtc_base:audio_format_to_string", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:event_tracer", + "../../rtc_base:gtest_prod", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:rtc_numerics", + "../../rtc_base:safe_conversions", + "../../rtc_base:safe_minmax", + "../../rtc_base:sanitizer", + "../../rtc_base/experiments:field_trial_parser", + "../../rtc_base/synchronization:mutex", + "../../system_wrappers", + "../../system_wrappers:field_trial", + "../../system_wrappers:metrics", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_source_set("default_neteq_factory") { + visibility += webrtc_default_visibility + sources = [ + "neteq/default_neteq_factory.cc", + "neteq/default_neteq_factory.h", + ] + deps = [ + ":neteq", + "../../api:scoped_refptr", + "../../api/audio_codecs:audio_codecs_api", + "../../api/neteq:default_neteq_controller_factory", + "../../api/neteq:neteq_api", + "../../system_wrappers:system_wrappers", + ] +} + +# Although providing only test support, this target must be outside of the +# rtc_include_tests conditional. The reason is that it supports fuzzer tests +# that ultimately are built and run as a part of the Chromium ecosystem, which +# does not set the rtc_include_tests flag. +rtc_library("neteq_tools_minimal") { + visibility += webrtc_default_visibility + sources = [ + "neteq/tools/audio_sink.cc", + "neteq/tools/audio_sink.h", + "neteq/tools/encode_neteq_input.cc", + "neteq/tools/encode_neteq_input.h", + "neteq/tools/neteq_input.cc", + "neteq/tools/neteq_input.h", + "neteq/tools/neteq_test.cc", + "neteq/tools/neteq_test.h", + "neteq/tools/packet.cc", + "neteq/tools/packet.h", + "neteq/tools/packet_source.cc", + "neteq/tools/packet_source.h", + ] + + deps = [ + ":default_neteq_factory", + ":neteq", + "../../api:array_view", + "../../api:neteq_simulator_api", + "../../api:rtp_headers", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/neteq:custom_neteq_factory", + "../../api/neteq:default_neteq_controller_factory", + "../../api/neteq:neteq_api", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:copy_on_write_buffer", + "../../rtc_base:safe_conversions", + "../../rtc_base:stringutils", + "../../system_wrappers", + "../rtp_rtcp:rtp_rtcp_format", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + defines = audio_codec_defines +} + +rtc_library("neteq_test_tools") { + visibility += webrtc_default_visibility + testonly = true + sources = [ + "neteq/tools/audio_checksum.h", + "neteq/tools/audio_loop.cc", + "neteq/tools/audio_loop.h", + "neteq/tools/constant_pcm_packet_source.cc", + "neteq/tools/constant_pcm_packet_source.h", + "neteq/tools/initial_packet_inserter_neteq_input.cc", + "neteq/tools/initial_packet_inserter_neteq_input.h", + "neteq/tools/neteq_packet_source_input.cc", + "neteq/tools/neteq_packet_source_input.h", + "neteq/tools/output_audio_file.h", + "neteq/tools/output_wav_file.h", + "neteq/tools/rtp_file_source.cc", + "neteq/tools/rtp_file_source.h", + "neteq/tools/rtp_generator.cc", + "neteq/tools/rtp_generator.h", + ] + + deps = [ + ":neteq_tools", + ":neteq_tools_minimal", + ":pcm16b", + "../../api:array_view", + "../../api:rtp_headers", + "../../common_audio", + "../../rtc_base:buffer", + "../../rtc_base:checks", + "../../rtc_base:ssl", + "../../rtc_base:stringutils", + "../../rtc_base/system:arch", + "../../test:rtp_test_utils", + "../rtp_rtcp:rtp_rtcp_format", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + + if (rtc_enable_protobuf) { + sources += [ + "neteq/tools/neteq_event_log_input.cc", + "neteq/tools/neteq_event_log_input.h", + ] + deps += [ ":rtc_event_log_source" ] + } +} + +rtc_library("neteq_tools") { + visibility += webrtc_default_visibility + sources = [ + "neteq/tools/fake_decode_from_file.cc", + "neteq/tools/fake_decode_from_file.h", + "neteq/tools/neteq_delay_analyzer.cc", + "neteq/tools/neteq_delay_analyzer.h", + "neteq/tools/neteq_replacement_input.cc", + "neteq/tools/neteq_replacement_input.h", + "neteq/tools/neteq_stats_getter.cc", + "neteq/tools/neteq_stats_getter.h", + "neteq/tools/neteq_stats_plotter.cc", + "neteq/tools/neteq_stats_plotter.h", + ] + + deps = [ + ":neteq_input_audio_tools", + ":neteq_tools_minimal", + "..:module_api_public", + "../../api:array_view", + "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:checks", + "../../rtc_base:rtc_numerics", + "../../rtc_base:safe_conversions", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", + "../rtp_rtcp", + "../rtp_rtcp:rtp_rtcp_format", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("neteq_input_audio_tools") { + visibility += webrtc_default_visibility + sources = [ + "neteq/tools/input_audio_file.cc", + "neteq/tools/input_audio_file.h", + "neteq/tools/resample_input_audio_file.cc", + "neteq/tools/resample_input_audio_file.h", + ] + + deps = [ + "../../common_audio", + "../../rtc_base:checks", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +if (rtc_enable_protobuf) { + rtc_library("rtc_event_log_source") { + testonly = true + + sources = [ + "neteq/tools/rtc_event_log_source.cc", + "neteq/tools/rtc_event_log_source.h", + ] + + deps = [ + ":neteq_tools_minimal", + "../../logging:rtc_event_log_parser", + "../../rtc_base:checks", + "../rtp_rtcp", + "../rtp_rtcp:rtp_rtcp_format", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + public_deps = # no-presubmit-check TODO(webrtc:8603) + [ "../../logging:rtc_event_log_proto" ] + } + + # Only used for test purpose. Since we want to use it from chromium + # (see audio_coding_modules_tests_shared below), we cannot guard it + # under rtc_include_tests. + proto_library("neteq_unittest_proto") { + testonly = true + sources = [ "neteq/neteq_unittest.proto" ] + proto_out_dir = "modules/audio_coding/neteq" + } +} + +# Allow to re-use some test classes from chromium. +rtc_library("audio_coding_modules_tests_shared") { + testonly = true + visibility = [] + visibility = [ "*" ] + + sources = [ + "neteq/test/neteq_decoding_test.cc", + "neteq/test/neteq_decoding_test.h", + "neteq/test/result_sink.cc", + "neteq/test/result_sink.h", + "test/PCMFile.cc", + "test/PCMFile.h", + "test/TestStereo.cc", + "test/TestStereo.h", + "test/opus_test.cc", + "test/opus_test.h", + ] + + deps = [ + ":audio_coding", + ":audio_coding_module_typedefs", + ":default_neteq_factory", + ":neteq_test_tools", + ":neteq_tools_minimal", + ":webrtc_opus_wrapper", + "..:module_api", + "../../api:rtp_headers", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../api/neteq:neteq_api", + "../../rtc_base:checks", + "../../rtc_base:ignore_wundef", + "../../rtc_base:ssl", + "../../rtc_base:stringutils", + "../../system_wrappers", + "../../test:fileutils", + "../../test:test_support", + "//testing/gtest", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + defines = audio_coding_defines + + if (rtc_enable_protobuf) { + defines += [ "WEBRTC_NETEQ_UNITTEST_BITEXACT" ] + deps += [ ":neteq_unittest_proto" ] + } +} + +if (rtc_include_tests) { + audio_coding_deps = [ + ":audio_encoder_cng", + ":g711", + ":g722", + ":pcm16b", + "../../common_audio", + "../../system_wrappers", + ] + if (rtc_include_ilbc) { + audio_coding_deps += [ ":ilbc" ] + } + if (rtc_include_opus) { + audio_coding_deps += [ ":webrtc_opus" ] + } + if (!build_with_mozilla && !build_with_chromium) { + audio_coding_deps += [ ":red" ] + } + + rtc_source_set("mocks") { + testonly = true + sources = [ + "audio_network_adaptor/mock/mock_audio_network_adaptor.h", + "audio_network_adaptor/mock/mock_controller.h", + "audio_network_adaptor/mock/mock_controller_manager.h", + "audio_network_adaptor/mock/mock_debug_dump_writer.h", + ] + deps = [ + ":audio_network_adaptor", + "../../test:test_support", + ] + } + + if (!build_with_chromium) { + group("audio_coding_tests") { + visibility += webrtc_default_visibility + testonly = true + public_deps = [ # no-presubmit-check TODO(webrtc:8603) + ":acm_receive_test", + ":acm_send_test", + ":audio_codec_speed_tests", + ":audio_decoder_unittests", + ":audio_decoder_unittests", + ":g711_test", + ":g722_test", + ":ilbc_test", + ":neteq_ilbc_quality_test", + ":neteq_opus_quality_test", + ":neteq_pcm16b_quality_test", + ":neteq_pcmu_quality_test", + ":neteq_speed_test", + ":rtp_analyze", + ":rtp_encode", + ":rtp_jitter", + ":rtpcat", + ":webrtc_opus_fec_test", + ] + if (rtc_enable_protobuf) { + public_deps += # no-presubmit-check TODO(webrtc:8603) + [ ":neteq_rtpplay" ] + } + } + } + + rtc_library("audio_coding_modules_tests") { + testonly = true + visibility += webrtc_default_visibility + + sources = [ + "test/Channel.cc", + "test/Channel.h", + "test/EncodeDecodeTest.cc", + "test/EncodeDecodeTest.h", + "test/PacketLossTest.cc", + "test/PacketLossTest.h", + "test/RTPFile.cc", + "test/RTPFile.h", + "test/TestAllCodecs.cc", + "test/TestAllCodecs.h", + "test/TestRedFec.cc", + "test/TestRedFec.h", + "test/TestVADDTX.cc", + "test/TestVADDTX.h", + "test/Tester.cc", + "test/TwoWayCommunication.cc", + "test/TwoWayCommunication.h", + "test/target_delay_unittest.cc", + ] + deps = [ + ":audio_coding", + ":audio_coding_module_typedefs", + ":audio_coding_modules_tests_shared", + ":audio_encoder_cng", + ":pcm16b_c", + ":red", + ":webrtc_opus_wrapper", + "..:module_api", + "../../api:rtp_headers", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../api/audio_codecs/L16:audio_decoder_L16", + "../../api/audio_codecs/L16:audio_encoder_L16", + "../../api/audio_codecs/g711:audio_decoder_g711", + "../../api/audio_codecs/g711:audio_encoder_g711", + "../../api/audio_codecs/g722:audio_decoder_g722", + "../../api/audio_codecs/g722:audio_encoder_g722", + "../../api/audio_codecs/ilbc:audio_decoder_ilbc", + "../../api/audio_codecs/ilbc:audio_encoder_ilbc", + "../../api/audio_codecs/opus:audio_decoder_opus", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../common_audio", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", + "../../rtc_base/synchronization:mutex", + "../../test:fileutils", + "../../test:scoped_key_value_config", + "../../test:test_support", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + defines = audio_coding_defines + } + + rtc_library("audio_coding_perf_tests") { + testonly = true + visibility += webrtc_default_visibility + + sources = [ + "codecs/opus/opus_complexity_unittest.cc", + "neteq/test/neteq_performance_unittest.cc", + ] + deps = [ + ":neteq_test_support", + ":neteq_test_tools", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../api/test/metrics:global_metrics_logger_and_exporter", + "../../api/test/metrics:metric", + "../../rtc_base:macromagic", + "../../rtc_base:timeutils", + "../../system_wrappers", + "../../system_wrappers:field_trial", + "../../test:fileutils", + "../../test:test_support", + ] + } + + rtc_library("acm_receive_test") { + testonly = true + sources = [ + "acm2/acm_receive_test.cc", + "acm2/acm_receive_test.h", + ] + + defines = audio_coding_defines + + deps = [ + ":audio_coding", + ":neteq_tools", + ":neteq_tools_minimal", + "../../api:scoped_refptr", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../test:test_support", + "//testing/gtest", + ] + + deps += audio_coding_deps + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("acm_send_test") { + testonly = true + sources = [ + "acm2/acm_send_test.cc", + "acm2/acm_send_test.h", + ] + + defines = audio_coding_defines + + deps = [ + ":audio_coding", + ":neteq_input_audio_tools", + ":neteq_tools", + ":neteq_tools_minimal", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../rtc_base:checks", + "../../rtc_base:stringutils", + "../../test:test_support", + "//testing/gtest", + ] + deps += audio_coding_deps + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + if (!build_with_chromium) { + audio_decoder_unittests_resources = + [ "../../resources/audio_coding/testfile32kHz.pcm" ] + + if (is_ios) { + bundle_data("audio_decoder_unittests_bundle_data") { + testonly = true + sources = audio_decoder_unittests_resources + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } + } + + rtc_test("audio_decoder_unittests") { + testonly = true + sources = [ "neteq/audio_decoder_unittest.cc" ] + + defines = neteq_defines + + deps = [ + ":ilbc", + ":neteq", + ":neteq_input_audio_tools", + ":neteq_tools", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../common_audio", + "../../rtc_base/system:arch", + "../../test:fileutils", + "../../test:test_main", + "../../test:test_support", + "//testing/gtest", + ] + audio_coding_deps + + data = audio_decoder_unittests_resources + + if (is_android) { + use_default_launcher = false + deps += [ + "//build/android/gtest_apk:native_test_instrumentation_test_runner_java", + "//testing/android/native_test:native_test_java", + "//testing/android/native_test:native_test_support", + ] + shard_timeout = 900 + } + if (is_ios) { + deps += [ ":audio_decoder_unittests_bundle_data" ] + } + } + } + + if (rtc_enable_protobuf) { + rtc_library("neteq_test_factory") { + testonly = true + visibility += webrtc_default_visibility + defines = audio_codec_defines + deps = [ + ":neteq_input_audio_tools", + ":neteq_tools", + ":neteq_tools_minimal", + "../../rtc_base:checks", + "../../rtc_base:refcount", + "../../test:fileutils", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + sources = [ + "neteq/tools/neteq_test_factory.cc", + "neteq/tools/neteq_test_factory.h", + ] + + deps += [ + ":neteq", + ":neteq_test_tools", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/neteq:neteq_api", + "../../test:audio_test_common", + "../../test:field_trial", + "../../test:test_support", + ] + } + } + + if (rtc_enable_protobuf && !build_with_chromium) { + rtc_executable("neteq_rtpplay") { + testonly = true + visibility += [ "*" ] + defines = [] + deps = [ + ":neteq_test_factory", + ":neteq_test_tools", + ":neteq_tools_minimal", + "../../rtc_base:stringutils", + "../../system_wrappers:field_trial", + "../../test:field_trial", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + sources = [ "neteq/tools/neteq_rtpplay.cc" ] + } + } + + if (!build_with_chromium) { + audio_codec_speed_tests_resources = [ + "//resources/audio_coding/music_stereo_48kHz.pcm", + "//resources/audio_coding/speech_mono_16kHz.pcm", + "//resources/audio_coding/speech_mono_32_48kHz.pcm", + ] + + if (is_ios) { + bundle_data("audio_codec_speed_tests_data") { + testonly = true + sources = audio_codec_speed_tests_resources + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } + } + + rtc_test("audio_codec_speed_tests") { + testonly = true + defines = [] + deps = [ + "../../rtc_base:macromagic", + "../../test:fileutils", + ] + sources = [ + "codecs/opus/opus_speed_test.cc", + "codecs/tools/audio_codec_speed_test.cc", + "codecs/tools/audio_codec_speed_test.h", + ] + + data = audio_codec_speed_tests_resources + + if (is_android) { + use_default_launcher = false + deps += [ + "//build/android/gtest_apk:native_test_instrumentation_test_runner_java", + "//testing/android/native_test:native_test_java", + "//testing/android/native_test:native_test_support", + ] + shard_timeout = 900 + } + + if (is_ios) { + deps += [ ":audio_codec_speed_tests_data" ] + } + + deps += [ + ":webrtc_opus", + "../../rtc_base:checks", + "../../test:test_main", + "../../test:test_support", + "../audio_processing", + "//testing/gtest", + ] + } + } + + rtc_library("neteq_test_support") { + testonly = true + sources = [ + "neteq/tools/neteq_performance_test.cc", + "neteq/tools/neteq_performance_test.h", + ] + + deps = [ + ":default_neteq_factory", + ":neteq", + ":neteq_test_tools", + ":pcm16b", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/neteq:neteq_api", + "../../rtc_base:checks", + "../../system_wrappers", + "../../test:fileutils", + "../../test:test_support", + "//testing/gtest", + ] + } + + if (!build_with_chromium) { + rtc_library("neteq_quality_test_support") { + testonly = true + sources = [ + "neteq/tools/neteq_quality_test.cc", + "neteq/tools/neteq_quality_test.h", + ] + + deps = [ + ":default_neteq_factory", + ":neteq", + ":neteq_input_audio_tools", + ":neteq_test_tools", + ":neteq_tools_minimal", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/neteq:neteq_api", + "../../rtc_base:checks", + "../../rtc_base:stringutils", + "../../system_wrappers", + "../../test:fileutils", + "../../test:test_support", + "//testing/gtest", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/strings", + ] + } + + rtc_executable("rtp_encode") { + testonly = true + + deps = [ + ":audio_coding", + ":audio_encoder_cng", + ":neteq_input_audio_tools", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs/L16:audio_encoder_L16", + "../../api/audio_codecs/g711:audio_encoder_g711", + "../../api/audio_codecs/g722:audio_encoder_g722", + "../../api/audio_codecs/ilbc:audio_encoder_ilbc", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../rtc_base:safe_conversions", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + "//third_party/abseil-cpp/absl/memory", + ] + + deps += audio_coding_deps + + sources = [ "neteq/tools/rtp_encode.cc" ] + + defines = audio_coding_defines + } + + rtc_executable("rtp_jitter") { + testonly = true + + deps = [ + "../../api:array_view", + "../../rtc_base:buffer", + "../rtp_rtcp:rtp_rtcp_format", + ] + + deps += audio_coding_deps + + sources = [ "neteq/tools/rtp_jitter.cc" ] + + defines = audio_coding_defines + } + + rtc_executable("rtpcat") { + testonly = true + + sources = [ "neteq/tools/rtpcat.cc" ] + + deps = [ + "../../rtc_base:checks", + "../../test:rtp_test_utils", + "//testing/gtest", + ] + } + + rtc_executable("rtp_analyze") { + testonly = true + + sources = [ "neteq/tools/rtp_analyze.cc" ] + + deps = [ + ":neteq", + ":neteq_test_tools", + ":neteq_tools_minimal", + ":pcm16b", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + ] + } + + rtc_executable("neteq_opus_quality_test") { + testonly = true + + sources = [ "neteq/test/neteq_opus_quality_test.cc" ] + + deps = [ + ":neteq", + ":neteq_quality_test_support", + ":neteq_tools", + ":webrtc_opus", + "../../test:test_main", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + ] + } + + rtc_executable("neteq_speed_test") { + testonly = true + + sources = [ "neteq/test/neteq_speed_test.cc" ] + + deps = [ + ":neteq", + ":neteq_test_support", + "../../rtc_base:checks", + "../../test:test_support", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + ] + } + + rtc_executable("neteq_ilbc_quality_test") { + testonly = true + + sources = [ "neteq/test/neteq_ilbc_quality_test.cc" ] + + deps = [ + ":ilbc", + ":neteq", + ":neteq_quality_test_support", + ":neteq_tools", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + "../../test:fileutils", + "../../test:test_main", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + ] + } + + rtc_executable("neteq_pcmu_quality_test") { + testonly = true + + sources = [ "neteq/test/neteq_pcmu_quality_test.cc" ] + + deps = [ + ":g711", + ":neteq", + ":neteq_quality_test_support", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + "../../test:fileutils", + "../../test:test_main", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + ] + } + + rtc_executable("neteq_pcm16b_quality_test") { + testonly = true + + sources = [ "neteq/test/neteq_pcm16b_quality_test.cc" ] + + deps = [ + ":neteq", + ":neteq_quality_test_support", + ":pcm16b", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + "../../test:fileutils", + "../../test:test_main", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + ] + } + } + + rtc_executable("g711_test") { + testonly = true + + sources = [ "codecs/g711/test/testG711.cc" ] + + deps = [ ":g711" ] + } + + rtc_executable("g722_test") { + testonly = true + + sources = [ "codecs/g722/test/testG722.cc" ] + + deps = [ ":g722" ] + } + + if (!build_with_chromium) { + rtc_executable("ilbc_test") { + testonly = true + + sources = [ "codecs/ilbc/test/iLBC_test.c" ] + + deps = [ ":ilbc" ] + } + + rtc_executable("webrtc_opus_fec_test") { + testonly = true + + sources = [ "codecs/opus/opus_fec_test.cc" ] + + deps = [ + ":webrtc_opus", + "../../common_audio", + "../../rtc_base:macromagic", + "../../test:fileutils", + "../../test:test_main", + "../../test:test_support", + "//testing/gtest", + ] + } + + rtc_library("audio_coding_unittests") { + testonly = true + visibility += webrtc_default_visibility + + sources = [ + "acm2/acm_receiver_unittest.cc", + "acm2/acm_remixing_unittest.cc", + "acm2/audio_coding_module_unittest.cc", + "acm2/call_statistics_unittest.cc", + "audio_network_adaptor/audio_network_adaptor_impl_unittest.cc", + "audio_network_adaptor/bitrate_controller_unittest.cc", + "audio_network_adaptor/channel_controller_unittest.cc", + "audio_network_adaptor/controller_manager_unittest.cc", + "audio_network_adaptor/dtx_controller_unittest.cc", + "audio_network_adaptor/event_log_writer_unittest.cc", + "audio_network_adaptor/fec_controller_plr_based_unittest.cc", + "audio_network_adaptor/frame_length_controller_unittest.cc", + "audio_network_adaptor/frame_length_controller_v2_unittest.cc", + "audio_network_adaptor/util/threshold_curve_unittest.cc", + "codecs/builtin_audio_decoder_factory_unittest.cc", + "codecs/builtin_audio_encoder_factory_unittest.cc", + "codecs/cng/audio_encoder_cng_unittest.cc", + "codecs/cng/cng_unittest.cc", + "codecs/ilbc/ilbc_unittest.cc", + "codecs/legacy_encoded_audio_frame_unittest.cc", + "codecs/opus/audio_decoder_multi_channel_opus_unittest.cc", + "codecs/opus/audio_encoder_multi_channel_opus_unittest.cc", + "codecs/opus/audio_encoder_opus_unittest.cc", + "codecs/opus/opus_bandwidth_unittest.cc", + "codecs/opus/opus_unittest.cc", + "codecs/red/audio_encoder_copy_red_unittest.cc", + "neteq/audio_multi_vector_unittest.cc", + "neteq/audio_vector_unittest.cc", + "neteq/background_noise_unittest.cc", + "neteq/buffer_level_filter_unittest.cc", + "neteq/comfort_noise_unittest.cc", + "neteq/decision_logic_unittest.cc", + "neteq/decoder_database_unittest.cc", + "neteq/delay_manager_unittest.cc", + "neteq/dsp_helper_unittest.cc", + "neteq/dtmf_buffer_unittest.cc", + "neteq/dtmf_tone_generator_unittest.cc", + "neteq/expand_unittest.cc", + "neteq/histogram_unittest.cc", + "neteq/merge_unittest.cc", + "neteq/mock/mock_buffer_level_filter.h", + "neteq/mock/mock_decoder_database.h", + "neteq/mock/mock_delay_manager.h", + "neteq/mock/mock_dtmf_buffer.h", + "neteq/mock/mock_dtmf_tone_generator.h", + "neteq/mock/mock_expand.h", + "neteq/mock/mock_histogram.h", + "neteq/mock/mock_neteq_controller.h", + "neteq/mock/mock_packet_buffer.h", + "neteq/mock/mock_red_payload_splitter.h", + "neteq/mock/mock_statistics_calculator.h", + "neteq/nack_tracker_unittest.cc", + "neteq/neteq_decoder_plc_unittest.cc", + "neteq/neteq_impl_unittest.cc", + "neteq/neteq_network_stats_unittest.cc", + "neteq/neteq_stereo_unittest.cc", + "neteq/neteq_unittest.cc", + "neteq/normal_unittest.cc", + "neteq/packet_arrival_history_unittest.cc", + "neteq/packet_buffer_unittest.cc", + "neteq/post_decode_vad_unittest.cc", + "neteq/random_vector_unittest.cc", + "neteq/red_payload_splitter_unittest.cc", + "neteq/reorder_optimizer_unittest.cc", + "neteq/statistics_calculator_unittest.cc", + "neteq/sync_buffer_unittest.cc", + "neteq/time_stretch_unittest.cc", + "neteq/timestamp_scaler_unittest.cc", + "neteq/tools/input_audio_file_unittest.cc", + "neteq/tools/packet_unittest.cc", + "neteq/underrun_optimizer_unittest.cc", + ] + + deps = [ + ":acm_receive_test", + ":acm_send_test", + ":audio_coding", + ":audio_coding_module_typedefs", + ":audio_coding_modules_tests_shared", + ":audio_coding_opus_common", + ":audio_encoder_cng", + ":audio_network_adaptor", + ":default_neteq_factory", + ":g711", + ":ilbc", + ":legacy_encoded_audio_frame", + ":mocks", + ":neteq", + ":neteq_input_audio_tools", + ":neteq_test_support", + ":neteq_test_tools", + ":neteq_tools", + ":neteq_tools_minimal", + ":pcm16b", + ":red", + ":webrtc_cng", + ":webrtc_opus", + "..:module_api", + "..:module_api_public", + "../../api:array_view", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../api/audio_codecs/opus:audio_decoder_multiopus", + "../../api/audio_codecs/opus:audio_decoder_opus", + "../../api/audio_codecs/opus:audio_encoder_multiopus", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../api/neteq:default_neteq_controller_factory", + "../../api/neteq:neteq_api", + "../../api/neteq:neteq_controller_api", + "../../api/neteq:tick_timer", + "../../api/neteq:tick_timer_unittest", + "../../api/rtc_event_log", + "../../common_audio", + "../../common_audio:common_audio_c", + "../../common_audio:mock_common_audio", + "../../logging:mocks", + "../../logging:rtc_event_audio", + "../../modules/rtp_rtcp:rtp_rtcp_format", + "../../rtc_base:checks", + "../../rtc_base:ignore_wundef", + "../../rtc_base:macromagic", + "../../rtc_base:platform_thread", + "../../rtc_base:refcount", + "../../rtc_base:rtc_base_tests_utils", + "../../rtc_base:rtc_event", + "../../rtc_base:safe_conversions", + "../../rtc_base:sanitizer", + "../../rtc_base:ssl", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", + "../../rtc_base/synchronization:mutex", + "../../rtc_base/system:arch", + "../../system_wrappers", + "../../test:audio_codec_mocks", + "../../test:audio_test_common", + "../../test:field_trial", + "../../test:fileutils", + "../../test:rtc_expect_death", + "../../test:rtp_test_utils", + "../../test:scoped_key_value_config", + "../../test:test_common", + "../../test:test_support", + "codecs/opus/test", + "codecs/opus/test:test_unittest", + "//testing/gtest", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + + defines = audio_coding_defines + + if (rtc_enable_protobuf) { + defines += [ "WEBRTC_NETEQ_UNITTEST_BITEXACT" ] + deps += [ + ":ana_config_proto", + ":neteq_unittest_proto", + ] + } + } + } +} + +# For backwards compatibility only! Use +# webrtc/api/audio_codecs:audio_codecs_api instead. +# TODO(kwiberg): Remove this. +rtc_source_set("audio_decoder_interface") { + visibility += [ "*" ] + sources = [ "codecs/audio_decoder.h" ] + deps = [ "../../api/audio_codecs:audio_codecs_api" ] +} + +# For backwards compatibility only! Use +# webrtc/api/audio_codecs:audio_codecs_api instead. +# TODO(ossu): Remove this. +rtc_source_set("audio_encoder_interface") { + visibility += [ "*" ] + sources = [ "codecs/audio_encoder.h" ] + deps = [ "../../api/audio_codecs:audio_codecs_api" ] +} diff --git a/third_party/libwebrtc/modules/audio_coding/DEPS b/third_party/libwebrtc/modules/audio_coding/DEPS new file mode 100644 index 0000000000..3dc9624a4b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/DEPS @@ -0,0 +1,7 @@ +include_rules = [ + "+call", + "+common_audio", + "+logging/rtc_event_log", + "+audio_coding/neteq/neteq_unittest.pb.h", # Different path. + "+system_wrappers", +] diff --git a/third_party/libwebrtc/modules/audio_coding/OWNERS b/third_party/libwebrtc/modules/audio_coding/OWNERS new file mode 100644 index 0000000000..c27c2a8d2d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/OWNERS @@ -0,0 +1,4 @@ +henrik.lundin@webrtc.org +minyue@webrtc.org +ivoc@webrtc.org +jakobi@webrtc.org diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_receive_test.cc b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receive_test.cc new file mode 100644 index 0000000000..8bc76cd2af --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receive_test.cc @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_receive_test.h" + +#include + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +AudioCodingModule::Config MakeAcmConfig( + Clock* clock, + rtc::scoped_refptr decoder_factory) { + AudioCodingModule::Config config; + config.clock = clock; + config.decoder_factory = std::move(decoder_factory); + return config; +} +} // namespace + +AcmReceiveTestOldApi::AcmReceiveTestOldApi( + PacketSource* packet_source, + AudioSink* audio_sink, + int output_freq_hz, + NumOutputChannels exptected_output_channels, + rtc::scoped_refptr decoder_factory) + : clock_(0), + acm_(webrtc::AudioCodingModule::Create( + MakeAcmConfig(&clock_, std::move(decoder_factory)))), + packet_source_(packet_source), + audio_sink_(audio_sink), + output_freq_hz_(output_freq_hz), + exptected_output_channels_(exptected_output_channels) {} + +AcmReceiveTestOldApi::~AcmReceiveTestOldApi() = default; + +void AcmReceiveTestOldApi::RegisterDefaultCodecs() { + acm_->SetReceiveCodecs({{103, {"ISAC", 16000, 1}}, + {104, {"ISAC", 32000, 1}}, + {107, {"L16", 8000, 1}}, + {108, {"L16", 16000, 1}}, + {109, {"L16", 32000, 1}}, + {111, {"L16", 8000, 2}}, + {112, {"L16", 16000, 2}}, + {113, {"L16", 32000, 2}}, + {0, {"PCMU", 8000, 1}}, + {110, {"PCMU", 8000, 2}}, + {8, {"PCMA", 8000, 1}}, + {118, {"PCMA", 8000, 2}}, + {102, {"ILBC", 8000, 1}}, + {9, {"G722", 8000, 1}}, + {119, {"G722", 8000, 2}}, + {120, {"OPUS", 48000, 2, {{"stereo", "1"}}}}, + {13, {"CN", 8000, 1}}, + {98, {"CN", 16000, 1}}, + {99, {"CN", 32000, 1}}}); +} + +// Remaps payload types from ACM's default to those used in the resource file +// neteq_universal_new.rtp. +void AcmReceiveTestOldApi::RegisterNetEqTestCodecs() { + acm_->SetReceiveCodecs({{103, {"ISAC", 16000, 1}}, + {104, {"ISAC", 32000, 1}}, + {93, {"L16", 8000, 1}}, + {94, {"L16", 16000, 1}}, + {95, {"L16", 32000, 1}}, + {0, {"PCMU", 8000, 1}}, + {8, {"PCMA", 8000, 1}}, + {102, {"ILBC", 8000, 1}}, + {9, {"G722", 8000, 1}}, + {120, {"OPUS", 48000, 2}}, + {13, {"CN", 8000, 1}}, + {98, {"CN", 16000, 1}}, + {99, {"CN", 32000, 1}}}); +} + +void AcmReceiveTestOldApi::Run() { + for (std::unique_ptr packet(packet_source_->NextPacket()); packet; + packet = packet_source_->NextPacket()) { + // Pull audio until time to insert packet. + while (clock_.TimeInMilliseconds() < packet->time_ms()) { + AudioFrame output_frame; + bool muted; + EXPECT_EQ(0, + acm_->PlayoutData10Ms(output_freq_hz_, &output_frame, &muted)); + ASSERT_EQ(output_freq_hz_, output_frame.sample_rate_hz_); + ASSERT_FALSE(muted); + const size_t samples_per_block = + static_cast(output_freq_hz_ * 10 / 1000); + EXPECT_EQ(samples_per_block, output_frame.samples_per_channel_); + if (exptected_output_channels_ != kArbitraryChannels) { + if (output_frame.speech_type_ == webrtc::AudioFrame::kPLC) { + // Don't check number of channels for PLC output, since each test run + // usually starts with a short period of mono PLC before decoding the + // first packet. + } else { + EXPECT_EQ(exptected_output_channels_, output_frame.num_channels_); + } + } + ASSERT_TRUE(audio_sink_->WriteAudioFrame(output_frame)); + clock_.AdvanceTimeMilliseconds(10); + AfterGetAudio(); + } + + EXPECT_EQ(0, acm_->IncomingPacket( + packet->payload(), + static_cast(packet->payload_length_bytes()), + packet->header())) + << "Failure when inserting packet:" << std::endl + << " PT = " << static_cast(packet->header().payloadType) + << std::endl + << " TS = " << packet->header().timestamp << std::endl + << " SN = " << packet->header().sequenceNumber; + } +} + +AcmReceiveTestToggleOutputFreqOldApi::AcmReceiveTestToggleOutputFreqOldApi( + PacketSource* packet_source, + AudioSink* audio_sink, + int output_freq_hz_1, + int output_freq_hz_2, + int toggle_period_ms, + NumOutputChannels exptected_output_channels) + : AcmReceiveTestOldApi(packet_source, + audio_sink, + output_freq_hz_1, + exptected_output_channels, + CreateBuiltinAudioDecoderFactory()), + output_freq_hz_1_(output_freq_hz_1), + output_freq_hz_2_(output_freq_hz_2), + toggle_period_ms_(toggle_period_ms), + last_toggle_time_ms_(clock_.TimeInMilliseconds()) {} + +void AcmReceiveTestToggleOutputFreqOldApi::AfterGetAudio() { + if (clock_.TimeInMilliseconds() >= last_toggle_time_ms_ + toggle_period_ms_) { + output_freq_hz_ = (output_freq_hz_ == output_freq_hz_1_) + ? output_freq_hz_2_ + : output_freq_hz_1_; + last_toggle_time_ms_ = clock_.TimeInMilliseconds(); + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_receive_test.h b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receive_test.h new file mode 100644 index 0000000000..2095ef9025 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receive_test.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_RECEIVE_TEST_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_RECEIVE_TEST_H_ + +#include // for size_t + +#include +#include + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/scoped_refptr.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +class AudioCodingModule; +class AudioDecoder; + +namespace test { +class AudioSink; +class PacketSource; + +class AcmReceiveTestOldApi { + public: + enum NumOutputChannels : size_t { + kArbitraryChannels = 0, + kMonoOutput = 1, + kStereoOutput = 2, + kQuadOutput = 4 + }; + + AcmReceiveTestOldApi(PacketSource* packet_source, + AudioSink* audio_sink, + int output_freq_hz, + NumOutputChannels exptected_output_channels, + rtc::scoped_refptr decoder_factory); + virtual ~AcmReceiveTestOldApi(); + + AcmReceiveTestOldApi(const AcmReceiveTestOldApi&) = delete; + AcmReceiveTestOldApi& operator=(const AcmReceiveTestOldApi&) = delete; + + // Registers the codecs with default parameters from ACM. + void RegisterDefaultCodecs(); + + // Registers codecs with payload types matching the pre-encoded NetEq test + // files. + void RegisterNetEqTestCodecs(); + + // Runs the test and returns true if successful. + void Run(); + + AudioCodingModule* get_acm() { return acm_.get(); } + + protected: + // Method is called after each block of output audio is received from ACM. + virtual void AfterGetAudio() {} + + SimulatedClock clock_; + std::unique_ptr acm_; + PacketSource* packet_source_; + AudioSink* audio_sink_; + int output_freq_hz_; + NumOutputChannels exptected_output_channels_; +}; + +// This test toggles the output frequency every `toggle_period_ms`. The test +// starts with `output_freq_hz_1`. Except for the toggling, it does the same +// thing as AcmReceiveTestOldApi. +class AcmReceiveTestToggleOutputFreqOldApi : public AcmReceiveTestOldApi { + public: + AcmReceiveTestToggleOutputFreqOldApi( + PacketSource* packet_source, + AudioSink* audio_sink, + int output_freq_hz_1, + int output_freq_hz_2, + int toggle_period_ms, + NumOutputChannels exptected_output_channels); + + protected: + void AfterGetAudio() override; + + const int output_freq_hz_1_; + const int output_freq_hz_2_; + const int toggle_period_ms_; + int64_t last_toggle_time_ms_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_ACM2_ACM_RECEIVE_TEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.cc b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.cc new file mode 100644 index 0000000000..b078af1d2d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.cc @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_receiver.h" + +#include +#include + +#include +#include + +#include "absl/strings/match.h" +#include "api/audio/audio_frame.h" +#include "api/audio_codecs/audio_decoder.h" +#include "api/neteq/neteq.h" +#include "modules/audio_coding/acm2/acm_resampler.h" +#include "modules/audio_coding/acm2/call_statistics.h" +#include "modules/audio_coding/neteq/default_neteq_factory.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/strings/audio_format_to_string.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +namespace acm2 { + +namespace { + +std::unique_ptr CreateNetEq( + NetEqFactory* neteq_factory, + const NetEq::Config& config, + Clock* clock, + const rtc::scoped_refptr& decoder_factory) { + if (neteq_factory) { + return neteq_factory->CreateNetEq(config, decoder_factory, clock); + } + return DefaultNetEqFactory().CreateNetEq(config, decoder_factory, clock); +} + +} // namespace + +AcmReceiver::AcmReceiver(const AudioCodingModule::Config& config) + : last_audio_buffer_(new int16_t[AudioFrame::kMaxDataSizeSamples]), + neteq_(CreateNetEq(config.neteq_factory, + config.neteq_config, + config.clock, + config.decoder_factory)), + clock_(config.clock), + resampled_last_output_frame_(true) { + RTC_DCHECK(clock_); + memset(last_audio_buffer_.get(), 0, + sizeof(int16_t) * AudioFrame::kMaxDataSizeSamples); +} + +AcmReceiver::~AcmReceiver() = default; + +int AcmReceiver::SetMinimumDelay(int delay_ms) { + if (neteq_->SetMinimumDelay(delay_ms)) + return 0; + RTC_LOG(LS_ERROR) << "AcmReceiver::SetExtraDelay " << delay_ms; + return -1; +} + +int AcmReceiver::SetMaximumDelay(int delay_ms) { + if (neteq_->SetMaximumDelay(delay_ms)) + return 0; + RTC_LOG(LS_ERROR) << "AcmReceiver::SetExtraDelay " << delay_ms; + return -1; +} + +bool AcmReceiver::SetBaseMinimumDelayMs(int delay_ms) { + return neteq_->SetBaseMinimumDelayMs(delay_ms); +} + +int AcmReceiver::GetBaseMinimumDelayMs() const { + return neteq_->GetBaseMinimumDelayMs(); +} + +absl::optional AcmReceiver::last_packet_sample_rate_hz() const { + MutexLock lock(&mutex_); + if (!last_decoder_) { + return absl::nullopt; + } + return last_decoder_->sample_rate_hz; +} + +int AcmReceiver::last_output_sample_rate_hz() const { + return neteq_->last_output_sample_rate_hz(); +} + +int AcmReceiver::InsertPacket(const RTPHeader& rtp_header, + rtc::ArrayView incoming_payload) { + if (incoming_payload.empty()) { + neteq_->InsertEmptyPacket(rtp_header); + return 0; + } + + int payload_type = rtp_header.payloadType; + auto format = neteq_->GetDecoderFormat(payload_type); + if (format && absl::EqualsIgnoreCase(format->sdp_format.name, "red")) { + // This is a RED packet. Get the format of the audio codec. + payload_type = incoming_payload[0] & 0x7f; + format = neteq_->GetDecoderFormat(payload_type); + } + if (!format) { + RTC_LOG_F(LS_ERROR) << "Payload-type " << payload_type + << " is not registered."; + return -1; + } + + { + MutexLock lock(&mutex_); + if (absl::EqualsIgnoreCase(format->sdp_format.name, "cn")) { + if (last_decoder_ && last_decoder_->num_channels > 1) { + // This is a CNG and the audio codec is not mono, so skip pushing in + // packets into NetEq. + return 0; + } + } else { + last_decoder_ = DecoderInfo{/*payload_type=*/payload_type, + /*sample_rate_hz=*/format->sample_rate_hz, + /*num_channels=*/format->num_channels, + /*sdp_format=*/std::move(format->sdp_format)}; + } + } // `mutex_` is released. + + if (neteq_->InsertPacket(rtp_header, incoming_payload) < 0) { + RTC_LOG(LS_ERROR) << "AcmReceiver::InsertPacket " + << static_cast(rtp_header.payloadType) + << " Failed to insert packet"; + return -1; + } + return 0; +} + +int AcmReceiver::GetAudio(int desired_freq_hz, + AudioFrame* audio_frame, + bool* muted) { + RTC_DCHECK(muted); + + int current_sample_rate_hz = 0; + if (neteq_->GetAudio(audio_frame, muted, ¤t_sample_rate_hz) != + NetEq::kOK) { + RTC_LOG(LS_ERROR) << "AcmReceiver::GetAudio - NetEq Failed."; + return -1; + } + + RTC_DCHECK_NE(current_sample_rate_hz, 0); + + // Update if resampling is required. + const bool need_resampling = + (desired_freq_hz != -1) && (current_sample_rate_hz != desired_freq_hz); + + // Accessing members, take the lock. + MutexLock lock(&mutex_); + if (need_resampling && !resampled_last_output_frame_) { + // Prime the resampler with the last frame. + int16_t temp_output[AudioFrame::kMaxDataSizeSamples]; + int samples_per_channel_int = resampler_.Resample10Msec( + last_audio_buffer_.get(), current_sample_rate_hz, desired_freq_hz, + audio_frame->num_channels_, AudioFrame::kMaxDataSizeSamples, + temp_output); + if (samples_per_channel_int < 0) { + RTC_LOG(LS_ERROR) << "AcmReceiver::GetAudio - " + "Resampling last_audio_buffer_ failed."; + return -1; + } + } + + // TODO(bugs.webrtc.org/3923) Glitches in the output may appear if the output + // rate from NetEq changes. + if (need_resampling) { + // TODO(yujo): handle this more efficiently for muted frames. + int samples_per_channel_int = resampler_.Resample10Msec( + audio_frame->data(), current_sample_rate_hz, desired_freq_hz, + audio_frame->num_channels_, AudioFrame::kMaxDataSizeSamples, + audio_frame->mutable_data()); + if (samples_per_channel_int < 0) { + RTC_LOG(LS_ERROR) + << "AcmReceiver::GetAudio - Resampling audio_buffer_ failed."; + return -1; + } + audio_frame->samples_per_channel_ = + static_cast(samples_per_channel_int); + audio_frame->sample_rate_hz_ = desired_freq_hz; + RTC_DCHECK_EQ( + audio_frame->sample_rate_hz_, + rtc::dchecked_cast(audio_frame->samples_per_channel_ * 100)); + resampled_last_output_frame_ = true; + } else { + resampled_last_output_frame_ = false; + // We might end up here ONLY if codec is changed. + } + + // Store current audio in `last_audio_buffer_` for next time. + memcpy(last_audio_buffer_.get(), audio_frame->data(), + sizeof(int16_t) * audio_frame->samples_per_channel_ * + audio_frame->num_channels_); + + call_stats_.DecodedByNetEq(audio_frame->speech_type_, *muted); + return 0; +} + +void AcmReceiver::SetCodecs(const std::map& codecs) { + neteq_->SetCodecs(codecs); +} + +void AcmReceiver::FlushBuffers() { + neteq_->FlushBuffers(); +} + +void AcmReceiver::RemoveAllCodecs() { + MutexLock lock(&mutex_); + neteq_->RemoveAllPayloadTypes(); + last_decoder_ = absl::nullopt; +} + +absl::optional AcmReceiver::GetPlayoutTimestamp() { + return neteq_->GetPlayoutTimestamp(); +} + +int AcmReceiver::FilteredCurrentDelayMs() const { + return neteq_->FilteredCurrentDelayMs(); +} + +int AcmReceiver::TargetDelayMs() const { + return neteq_->TargetDelayMs(); +} + +absl::optional> AcmReceiver::LastDecoder() + const { + MutexLock lock(&mutex_); + if (!last_decoder_) { + return absl::nullopt; + } + RTC_DCHECK_NE(-1, last_decoder_->payload_type); + return std::make_pair(last_decoder_->payload_type, last_decoder_->sdp_format); +} + +void AcmReceiver::GetNetworkStatistics( + NetworkStatistics* acm_stat, + bool get_and_clear_legacy_stats /* = true */) const { + NetEqNetworkStatistics neteq_stat; + if (get_and_clear_legacy_stats) { + // NetEq function always returns zero, so we don't check the return value. + neteq_->NetworkStatistics(&neteq_stat); + + acm_stat->currentExpandRate = neteq_stat.expand_rate; + acm_stat->currentSpeechExpandRate = neteq_stat.speech_expand_rate; + acm_stat->currentPreemptiveRate = neteq_stat.preemptive_rate; + acm_stat->currentAccelerateRate = neteq_stat.accelerate_rate; + acm_stat->currentSecondaryDecodedRate = neteq_stat.secondary_decoded_rate; + acm_stat->currentSecondaryDiscardedRate = + neteq_stat.secondary_discarded_rate; + acm_stat->meanWaitingTimeMs = neteq_stat.mean_waiting_time_ms; + acm_stat->maxWaitingTimeMs = neteq_stat.max_waiting_time_ms; + } else { + neteq_stat = neteq_->CurrentNetworkStatistics(); + acm_stat->currentExpandRate = 0; + acm_stat->currentSpeechExpandRate = 0; + acm_stat->currentPreemptiveRate = 0; + acm_stat->currentAccelerateRate = 0; + acm_stat->currentSecondaryDecodedRate = 0; + acm_stat->currentSecondaryDiscardedRate = 0; + acm_stat->meanWaitingTimeMs = -1; + acm_stat->maxWaitingTimeMs = 1; + } + acm_stat->currentBufferSize = neteq_stat.current_buffer_size_ms; + acm_stat->preferredBufferSize = neteq_stat.preferred_buffer_size_ms; + acm_stat->jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false; + + NetEqLifetimeStatistics neteq_lifetime_stat = neteq_->GetLifetimeStatistics(); + acm_stat->totalSamplesReceived = neteq_lifetime_stat.total_samples_received; + acm_stat->concealedSamples = neteq_lifetime_stat.concealed_samples; + acm_stat->silentConcealedSamples = + neteq_lifetime_stat.silent_concealed_samples; + acm_stat->concealmentEvents = neteq_lifetime_stat.concealment_events; + acm_stat->jitterBufferDelayMs = neteq_lifetime_stat.jitter_buffer_delay_ms; + acm_stat->jitterBufferTargetDelayMs = + neteq_lifetime_stat.jitter_buffer_target_delay_ms; + acm_stat->jitterBufferMinimumDelayMs = + neteq_lifetime_stat.jitter_buffer_minimum_delay_ms; + acm_stat->jitterBufferEmittedCount = + neteq_lifetime_stat.jitter_buffer_emitted_count; + acm_stat->delayedPacketOutageSamples = + neteq_lifetime_stat.delayed_packet_outage_samples; + acm_stat->relativePacketArrivalDelayMs = + neteq_lifetime_stat.relative_packet_arrival_delay_ms; + acm_stat->interruptionCount = neteq_lifetime_stat.interruption_count; + acm_stat->totalInterruptionDurationMs = + neteq_lifetime_stat.total_interruption_duration_ms; + acm_stat->insertedSamplesForDeceleration = + neteq_lifetime_stat.inserted_samples_for_deceleration; + acm_stat->removedSamplesForAcceleration = + neteq_lifetime_stat.removed_samples_for_acceleration; + acm_stat->fecPacketsReceived = neteq_lifetime_stat.fec_packets_received; + acm_stat->fecPacketsDiscarded = neteq_lifetime_stat.fec_packets_discarded; + acm_stat->packetsDiscarded = neteq_lifetime_stat.packets_discarded; + + NetEqOperationsAndState neteq_operations_and_state = + neteq_->GetOperationsAndState(); + acm_stat->packetBufferFlushes = + neteq_operations_and_state.packet_buffer_flushes; +} + +int AcmReceiver::EnableNack(size_t max_nack_list_size) { + neteq_->EnableNack(max_nack_list_size); + return 0; +} + +void AcmReceiver::DisableNack() { + neteq_->DisableNack(); +} + +std::vector AcmReceiver::GetNackList( + int64_t round_trip_time_ms) const { + return neteq_->GetNackList(round_trip_time_ms); +} + +void AcmReceiver::ResetInitialDelay() { + neteq_->SetMinimumDelay(0); + // TODO(turajs): Should NetEq Buffer be flushed? +} + +uint32_t AcmReceiver::NowInTimestamp(int decoder_sampling_rate) const { + // Down-cast the time to (32-6)-bit since we only care about + // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms. + // We masked 6 most significant bits of 32-bit so there is no overflow in + // the conversion from milliseconds to timestamp. + const uint32_t now_in_ms = + static_cast(clock_->TimeInMilliseconds() & 0x03ffffff); + return static_cast((decoder_sampling_rate / 1000) * now_in_ms); +} + +void AcmReceiver::GetDecodingCallStatistics( + AudioDecodingCallStats* stats) const { + MutexLock lock(&mutex_); + *stats = call_stats_.GetDecodingStatistics(); +} + +} // namespace acm2 + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.h b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.h new file mode 100644 index 0000000000..a61247627f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.h @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_RECEIVER_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_RECEIVER_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/audio_codecs/audio_decoder.h" +#include "api/audio_codecs/audio_format.h" +#include "modules/audio_coding/acm2/acm_resampler.h" +#include "modules/audio_coding/acm2/call_statistics.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +class Clock; +class NetEq; +struct RTPHeader; + +namespace acm2 { + +class AcmReceiver { + public: + // Constructor of the class + explicit AcmReceiver(const AudioCodingModule::Config& config); + + // Destructor of the class. + ~AcmReceiver(); + + // + // Inserts a payload with its associated RTP-header into NetEq. + // + // Input: + // - rtp_header : RTP header for the incoming payload containing + // information about payload type, sequence number, + // timestamp, SSRC and marker bit. + // - incoming_payload : Incoming audio payload. + // - length_payload : Length of incoming audio payload in bytes. + // + // Return value : 0 if OK. + // <0 if NetEq returned an error. + // + int InsertPacket(const RTPHeader& rtp_header, + rtc::ArrayView incoming_payload); + + // + // Asks NetEq for 10 milliseconds of decoded audio. + // + // Input: + // -desired_freq_hz : specifies the sampling rate [Hz] of the output + // audio. If set -1 indicates to resampling is + // is required and the audio returned at the + // sampling rate of the decoder. + // + // Output: + // -audio_frame : an audio frame were output data and + // associated parameters are written to. + // -muted : if true, the sample data in audio_frame is not + // populated, and must be interpreted as all zero. + // + // Return value : 0 if OK. + // -1 if NetEq returned an error. + // + int GetAudio(int desired_freq_hz, AudioFrame* audio_frame, bool* muted); + + // Replace the current set of decoders with the specified set. + void SetCodecs(const std::map& codecs); + + // + // Sets a minimum delay for packet buffer. The given delay is maintained, + // unless channel condition dictates a higher delay. + // + // Input: + // - delay_ms : minimum delay in milliseconds. + // + // Return value : 0 if OK. + // <0 if NetEq returned an error. + // + int SetMinimumDelay(int delay_ms); + + // + // Sets a maximum delay [ms] for the packet buffer. The target delay does not + // exceed the given value, even if channel condition requires so. + // + // Input: + // - delay_ms : maximum delay in milliseconds. + // + // Return value : 0 if OK. + // <0 if NetEq returned an error. + // + int SetMaximumDelay(int delay_ms); + + // Sets a base minimum delay in milliseconds for the packet buffer. + // Base minimum delay sets lower bound minimum delay value which + // is set via SetMinimumDelay. + // + // Returns true if value was successfully set, false overwise. + bool SetBaseMinimumDelayMs(int delay_ms); + + // Returns current value of base minimum delay in milliseconds. + int GetBaseMinimumDelayMs() const; + + // + // Resets the initial delay to zero. + // + void ResetInitialDelay(); + + // Returns the sample rate of the decoder associated with the last incoming + // packet. If no packet of a registered non-CNG codec has been received, the + // return value is empty. Also, if the decoder was unregistered since the last + // packet was inserted, the return value is empty. + absl::optional last_packet_sample_rate_hz() const; + + // Returns last_output_sample_rate_hz from the NetEq instance. + int last_output_sample_rate_hz() const; + + // + // Get the current network statistics from NetEq. + // + // Output: + // - statistics : The current network statistics. + // + void GetNetworkStatistics(NetworkStatistics* statistics, + bool get_and_clear_legacy_stats = true) const; + + // + // Flushes the NetEq packet and speech buffers. + // + void FlushBuffers(); + + // + // Remove all registered codecs. + // + void RemoveAllCodecs(); + + // Returns the RTP timestamp for the last sample delivered by GetAudio(). + // The return value will be empty if no valid timestamp is available. + absl::optional GetPlayoutTimestamp(); + + // Returns the current total delay from NetEq (packet buffer and sync buffer) + // in ms, with smoothing applied to even out short-time fluctuations due to + // jitter. The packet buffer part of the delay is not updated during DTX/CNG + // periods. + // + int FilteredCurrentDelayMs() const; + + // Returns the current target delay for NetEq in ms. + // + int TargetDelayMs() const; + + // + // Get payload type and format of the last non-CNG/non-DTMF received payload. + // If no non-CNG/non-DTMF packet is received absl::nullopt is returned. + // + absl::optional> LastDecoder() const; + + // + // Enable NACK and set the maximum size of the NACK list. If NACK is already + // enabled then the maximum NACK list size is modified accordingly. + // + // If the sequence number of last received packet is N, the sequence numbers + // of NACK list are in the range of [N - `max_nack_list_size`, N). + // + // `max_nack_list_size` should be positive (none zero) and less than or + // equal to `Nack::kNackListSizeLimit`. Otherwise, No change is applied and -1 + // is returned. 0 is returned at success. + // + int EnableNack(size_t max_nack_list_size); + + // Disable NACK. + void DisableNack(); + + // + // Get a list of packets to be retransmitted. `round_trip_time_ms` is an + // estimate of the round-trip-time (in milliseconds). Missing packets which + // will be playout in a shorter time than the round-trip-time (with respect + // to the time this API is called) will not be included in the list. + // + // Negative `round_trip_time_ms` results is an error message and empty list + // is returned. + // + std::vector GetNackList(int64_t round_trip_time_ms) const; + + // + // Get statistics of calls to GetAudio(). + void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const; + + private: + struct DecoderInfo { + int payload_type; + int sample_rate_hz; + int num_channels; + SdpAudioFormat sdp_format; + }; + + uint32_t NowInTimestamp(int decoder_sampling_rate) const; + + mutable Mutex mutex_; + absl::optional last_decoder_ RTC_GUARDED_BY(mutex_); + ACMResampler resampler_; + + // After construction, this is only ever touched on the thread that calls + // AcmReceiver::GetAudio, and only modified in this method. + std::unique_ptr last_audio_buffer_; + CallStatistics call_stats_; + const std::unique_ptr neteq_; // NetEq is thread-safe; no lock needed. + Clock* const clock_; + std::atomic resampled_last_output_frame_; +}; + +} // namespace acm2 + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_ACM_RECEIVER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver_unittest.cc b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver_unittest.cc new file mode 100644 index 0000000000..6dd44b696e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver_unittest.cc @@ -0,0 +1,455 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_receiver.h" + +#include // std::min +#include + +#include "absl/types/optional.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +namespace acm2 { + +class AcmReceiverTestOldApi : public AudioPacketizationCallback, + public ::testing::Test { + protected: + AcmReceiverTestOldApi() + : timestamp_(0), + packet_sent_(false), + last_packet_send_timestamp_(timestamp_), + last_frame_type_(AudioFrameType::kEmptyFrame) { + config_.decoder_factory = decoder_factory_; + } + + ~AcmReceiverTestOldApi() {} + + void SetUp() override { + acm_.reset(AudioCodingModule::Create(config_)); + receiver_.reset(new AcmReceiver(config_)); + ASSERT_TRUE(receiver_.get() != NULL); + ASSERT_TRUE(acm_.get() != NULL); + acm_->InitializeReceiver(); + acm_->RegisterTransportCallback(this); + + rtp_header_.sequenceNumber = 0; + rtp_header_.timestamp = 0; + rtp_header_.markerBit = false; + rtp_header_.ssrc = 0x12345678; // Arbitrary. + rtp_header_.numCSRCs = 0; + rtp_header_.payloadType = 0; + } + + void TearDown() override {} + + AudioCodecInfo SetEncoder(int payload_type, + const SdpAudioFormat& format, + const std::map cng_payload_types = {}) { + // Create the speech encoder. + absl::optional info = + encoder_factory_->QueryAudioEncoder(format); + RTC_CHECK(info.has_value()); + std::unique_ptr enc = + encoder_factory_->MakeAudioEncoder(payload_type, format, absl::nullopt); + + // If we have a compatible CN specification, stack a CNG on top. + auto it = cng_payload_types.find(info->sample_rate_hz); + if (it != cng_payload_types.end()) { + AudioEncoderCngConfig config; + config.speech_encoder = std::move(enc); + config.num_channels = 1; + config.payload_type = it->second; + config.vad_mode = Vad::kVadNormal; + enc = CreateComfortNoiseEncoder(std::move(config)); + } + + // Actually start using the new encoder. + acm_->SetEncoder(std::move(enc)); + return *info; + } + + int InsertOnePacketOfSilence(const AudioCodecInfo& info) { + // Frame setup according to the codec. + AudioFrame frame; + frame.sample_rate_hz_ = info.sample_rate_hz; + frame.samples_per_channel_ = info.sample_rate_hz / 100; // 10 ms. + frame.num_channels_ = info.num_channels; + frame.Mute(); + packet_sent_ = false; + last_packet_send_timestamp_ = timestamp_; + int num_10ms_frames = 0; + while (!packet_sent_) { + frame.timestamp_ = timestamp_; + timestamp_ += rtc::checked_cast(frame.samples_per_channel_); + EXPECT_GE(acm_->Add10MsData(frame), 0); + ++num_10ms_frames; + } + return num_10ms_frames; + } + + int SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + int64_t absolute_capture_timestamp_ms) override { + if (frame_type == AudioFrameType::kEmptyFrame) + return 0; + + rtp_header_.payloadType = payload_type; + rtp_header_.timestamp = timestamp; + + int ret_val = receiver_->InsertPacket( + rtp_header_, + rtc::ArrayView(payload_data, payload_len_bytes)); + if (ret_val < 0) { + RTC_DCHECK_NOTREACHED(); + return -1; + } + rtp_header_.sequenceNumber++; + packet_sent_ = true; + last_frame_type_ = frame_type; + return 0; + } + + const rtc::scoped_refptr encoder_factory_ = + CreateBuiltinAudioEncoderFactory(); + const rtc::scoped_refptr decoder_factory_ = + CreateBuiltinAudioDecoderFactory(); + AudioCodingModule::Config config_; + std::unique_ptr receiver_; + std::unique_ptr acm_; + RTPHeader rtp_header_; + uint32_t timestamp_; + bool packet_sent_; // Set when SendData is called reset when inserting audio. + uint32_t last_packet_send_timestamp_; + AudioFrameType last_frame_type_; +}; + +#if defined(WEBRTC_ANDROID) +#define MAYBE_SampleRate DISABLED_SampleRate +#else +#define MAYBE_SampleRate SampleRate +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_SampleRate) { + const std::map codecs = {{0, {"OPUS", 48000, 2}}}; + receiver_->SetCodecs(codecs); + + constexpr int kOutSampleRateHz = 8000; // Different than codec sample rate. + for (size_t i = 0; i < codecs.size(); ++i) { + const int payload_type = rtc::checked_cast(i); + const int num_10ms_frames = + InsertOnePacketOfSilence(SetEncoder(payload_type, codecs.at(i))); + for (int k = 0; k < num_10ms_frames; ++k) { + AudioFrame frame; + bool muted; + EXPECT_EQ(0, receiver_->GetAudio(kOutSampleRateHz, &frame, &muted)); + } + EXPECT_EQ(encoder_factory_->QueryAudioEncoder(codecs.at(i))->sample_rate_hz, + receiver_->last_output_sample_rate_hz()); + } +} + +class AcmReceiverTestFaxModeOldApi : public AcmReceiverTestOldApi { + protected: + AcmReceiverTestFaxModeOldApi() { + config_.neteq_config.for_test_no_time_stretching = true; + } + + void RunVerifyAudioFrame(const SdpAudioFormat& codec) { + // Make sure "fax mode" is enabled. This will avoid delay changes unless the + // packet-loss concealment is made. We do this in order to make the + // timestamp increments predictable; in normal mode, NetEq may decide to do + // accelerate or pre-emptive expand operations after some time, offsetting + // the timestamp. + EXPECT_TRUE(config_.neteq_config.for_test_no_time_stretching); + + constexpr int payload_type = 17; + receiver_->SetCodecs({{payload_type, codec}}); + + const AudioCodecInfo info = SetEncoder(payload_type, codec); + const int output_sample_rate_hz = info.sample_rate_hz; + const size_t output_channels = info.num_channels; + const size_t samples_per_ms = rtc::checked_cast( + rtc::CheckedDivExact(output_sample_rate_hz, 1000)); + const AudioFrame::VADActivity expected_vad_activity = + output_sample_rate_hz > 16000 ? AudioFrame::kVadActive + : AudioFrame::kVadPassive; + + // Expect the first output timestamp to be 5*fs/8000 samples before the + // first inserted timestamp (because of NetEq's look-ahead). (This value is + // defined in Expand::overlap_length_.) + uint32_t expected_output_ts = + last_packet_send_timestamp_ - + rtc::CheckedDivExact(5 * output_sample_rate_hz, 8000); + + AudioFrame frame; + bool muted; + EXPECT_EQ(0, receiver_->GetAudio(output_sample_rate_hz, &frame, &muted)); + // Expect timestamp = 0 before first packet is inserted. + EXPECT_EQ(0u, frame.timestamp_); + for (int i = 0; i < 5; ++i) { + const int num_10ms_frames = InsertOnePacketOfSilence(info); + for (int k = 0; k < num_10ms_frames; ++k) { + EXPECT_EQ(0, + receiver_->GetAudio(output_sample_rate_hz, &frame, &muted)); + EXPECT_EQ(expected_output_ts, frame.timestamp_); + expected_output_ts += rtc::checked_cast(10 * samples_per_ms); + EXPECT_EQ(10 * samples_per_ms, frame.samples_per_channel_); + EXPECT_EQ(output_sample_rate_hz, frame.sample_rate_hz_); + EXPECT_EQ(output_channels, frame.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, frame.speech_type_); + EXPECT_EQ(expected_vad_activity, frame.vad_activity_); + EXPECT_FALSE(muted); + } + } + } +}; + +#if defined(WEBRTC_ANDROID) +#define MAYBE_VerifyAudioFramePCMU DISABLED_VerifyAudioFramePCMU +#else +#define MAYBE_VerifyAudioFramePCMU VerifyAudioFramePCMU +#endif +TEST_F(AcmReceiverTestFaxModeOldApi, MAYBE_VerifyAudioFramePCMU) { + RunVerifyAudioFrame({"PCMU", 8000, 1}); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_VerifyAudioFrameOpus DISABLED_VerifyAudioFrameOpus +#else +#define MAYBE_VerifyAudioFrameOpus VerifyAudioFrameOpus +#endif +TEST_F(AcmReceiverTestFaxModeOldApi, MAYBE_VerifyAudioFrameOpus) { + RunVerifyAudioFrame({"opus", 48000, 2}); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_PostdecodingVad DISABLED_PostdecodingVad +#else +#define MAYBE_PostdecodingVad PostdecodingVad +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_PostdecodingVad) { + EXPECT_TRUE(config_.neteq_config.enable_post_decode_vad); + constexpr int payload_type = 34; + const SdpAudioFormat codec = {"L16", 16000, 1}; + const AudioCodecInfo info = SetEncoder(payload_type, codec); + receiver_->SetCodecs({{payload_type, codec}}); + constexpr int kNumPackets = 5; + AudioFrame frame; + for (int n = 0; n < kNumPackets; ++n) { + const int num_10ms_frames = InsertOnePacketOfSilence(info); + for (int k = 0; k < num_10ms_frames; ++k) { + bool muted; + ASSERT_EQ(0, receiver_->GetAudio(info.sample_rate_hz, &frame, &muted)); + } + } + EXPECT_EQ(AudioFrame::kVadPassive, frame.vad_activity_); +} + +class AcmReceiverTestPostDecodeVadPassiveOldApi : public AcmReceiverTestOldApi { + protected: + AcmReceiverTestPostDecodeVadPassiveOldApi() { + config_.neteq_config.enable_post_decode_vad = false; + } +}; + +#if defined(WEBRTC_ANDROID) +#define MAYBE_PostdecodingVad DISABLED_PostdecodingVad +#else +#define MAYBE_PostdecodingVad PostdecodingVad +#endif +TEST_F(AcmReceiverTestPostDecodeVadPassiveOldApi, MAYBE_PostdecodingVad) { + EXPECT_FALSE(config_.neteq_config.enable_post_decode_vad); + constexpr int payload_type = 34; + const SdpAudioFormat codec = {"L16", 16000, 1}; + const AudioCodecInfo info = SetEncoder(payload_type, codec); + auto const value = encoder_factory_->QueryAudioEncoder(codec); + ASSERT_TRUE(value.has_value()); + receiver_->SetCodecs({{payload_type, codec}}); + const int kNumPackets = 5; + AudioFrame frame; + for (int n = 0; n < kNumPackets; ++n) { + const int num_10ms_frames = InsertOnePacketOfSilence(info); + for (int k = 0; k < num_10ms_frames; ++k) { + bool muted; + ASSERT_EQ(0, receiver_->GetAudio(info.sample_rate_hz, &frame, &muted)); + } + } + EXPECT_EQ(AudioFrame::kVadUnknown, frame.vad_activity_); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_LastAudioCodec DISABLED_LastAudioCodec +#else +#define MAYBE_LastAudioCodec LastAudioCodec +#endif +#if defined(WEBRTC_CODEC_OPUS) +TEST_F(AcmReceiverTestOldApi, MAYBE_LastAudioCodec) { + const std::map codecs = { + {0, {"PCMU", 8000, 1}}, {1, {"PCMA", 8000, 1}}, {2, {"L16", 32000, 1}}}; + const std::map cng_payload_types = { + {8000, 100}, {16000, 101}, {32000, 102}}; + { + std::map receive_codecs = codecs; + for (const auto& cng_type : cng_payload_types) { + receive_codecs.emplace(std::make_pair( + cng_type.second, SdpAudioFormat("CN", cng_type.first, 1))); + } + receiver_->SetCodecs(receive_codecs); + } + + // No audio payload is received. + EXPECT_EQ(absl::nullopt, receiver_->LastDecoder()); + + // Start with sending DTX. + packet_sent_ = false; + InsertOnePacketOfSilence( + SetEncoder(0, codecs.at(0), cng_payload_types)); // Enough to test + // with one codec. + ASSERT_TRUE(packet_sent_); + EXPECT_EQ(AudioFrameType::kAudioFrameCN, last_frame_type_); + + // Has received, only, DTX. Last Audio codec is undefined. + EXPECT_EQ(absl::nullopt, receiver_->LastDecoder()); + EXPECT_EQ(absl::nullopt, receiver_->last_packet_sample_rate_hz()); + + for (size_t i = 0; i < codecs.size(); ++i) { + // Set DTX off to send audio payload. + packet_sent_ = false; + const int payload_type = rtc::checked_cast(i); + const AudioCodecInfo info_without_cng = + SetEncoder(payload_type, codecs.at(i)); + InsertOnePacketOfSilence(info_without_cng); + + // Sanity check if Actually an audio payload received, and it should be + // of type "speech." + ASSERT_TRUE(packet_sent_); + ASSERT_EQ(AudioFrameType::kAudioFrameSpeech, last_frame_type_); + EXPECT_EQ(info_without_cng.sample_rate_hz, + receiver_->last_packet_sample_rate_hz()); + + // Set VAD on to send DTX. Then check if the "Last Audio codec" returns + // the expected codec. Encode repeatedly until a DTX is sent. + const AudioCodecInfo info_with_cng = + SetEncoder(payload_type, codecs.at(i), cng_payload_types); + while (last_frame_type_ != AudioFrameType::kAudioFrameCN) { + packet_sent_ = false; + InsertOnePacketOfSilence(info_with_cng); + ASSERT_TRUE(packet_sent_); + } + EXPECT_EQ(info_with_cng.sample_rate_hz, + receiver_->last_packet_sample_rate_hz()); + EXPECT_EQ(codecs.at(i), receiver_->LastDecoder()->second); + } +} +#endif + +// Check if the statistics are initialized correctly. Before any call to ACM +// all fields have to be zero. +#if defined(WEBRTC_ANDROID) +#define MAYBE_InitializedToZero DISABLED_InitializedToZero +#else +#define MAYBE_InitializedToZero InitializedToZero +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_InitializedToZero) { + AudioDecodingCallStats stats; + receiver_->GetDecodingCallStatistics(&stats); + EXPECT_EQ(0, stats.calls_to_neteq); + EXPECT_EQ(0, stats.calls_to_silence_generator); + EXPECT_EQ(0, stats.decoded_normal); + EXPECT_EQ(0, stats.decoded_cng); + EXPECT_EQ(0, stats.decoded_neteq_plc); + EXPECT_EQ(0, stats.decoded_plc_cng); + EXPECT_EQ(0, stats.decoded_muted_output); +} + +// Insert some packets and pull audio. Check statistics are valid. Then, +// simulate packet loss and check if PLC and PLC-to-CNG statistics are +// correctly updated. +#if defined(WEBRTC_ANDROID) +#define MAYBE_NetEqCalls DISABLED_NetEqCalls +#else +#define MAYBE_NetEqCalls NetEqCalls +#endif +TEST_F(AcmReceiverTestOldApi, MAYBE_NetEqCalls) { + AudioDecodingCallStats stats; + const int kNumNormalCalls = 10; + const int kSampleRateHz = 16000; + const int kNumSamples10ms = kSampleRateHz / 100; + const int kFrameSizeMs = 10; // Multiple of 10. + const int kFrameSizeSamples = kFrameSizeMs / 10 * kNumSamples10ms; + const int kPayloadSizeBytes = kFrameSizeSamples * sizeof(int16_t); + const uint8_t kPayloadType = 111; + RTPHeader rtp_header; + AudioFrame audio_frame; + bool muted; + + receiver_->SetCodecs( + {{kPayloadType, SdpAudioFormat("L16", kSampleRateHz, 1)}}); + rtp_header.sequenceNumber = 0xABCD; + rtp_header.timestamp = 0xABCDEF01; + rtp_header.payloadType = kPayloadType; + rtp_header.markerBit = false; + rtp_header.ssrc = 0x1234; + rtp_header.numCSRCs = 0; + rtp_header.payload_type_frequency = kSampleRateHz; + + for (int num_calls = 0; num_calls < kNumNormalCalls; ++num_calls) { + const uint8_t kPayload[kPayloadSizeBytes] = {0}; + ASSERT_EQ(0, receiver_->InsertPacket(rtp_header, kPayload)); + ++rtp_header.sequenceNumber; + rtp_header.timestamp += kFrameSizeSamples; + ASSERT_EQ(0, receiver_->GetAudio(-1, &audio_frame, &muted)); + EXPECT_FALSE(muted); + } + receiver_->GetDecodingCallStatistics(&stats); + EXPECT_EQ(kNumNormalCalls, stats.calls_to_neteq); + EXPECT_EQ(0, stats.calls_to_silence_generator); + EXPECT_EQ(kNumNormalCalls, stats.decoded_normal); + EXPECT_EQ(0, stats.decoded_cng); + EXPECT_EQ(0, stats.decoded_neteq_plc); + EXPECT_EQ(0, stats.decoded_plc_cng); + EXPECT_EQ(0, stats.decoded_muted_output); + + const int kNumPlc = 3; + const int kNumPlcCng = 5; + + // Simulate packet-loss. NetEq first performs PLC then PLC fades to CNG. + for (int n = 0; n < kNumPlc + kNumPlcCng; ++n) { + ASSERT_EQ(0, receiver_->GetAudio(-1, &audio_frame, &muted)); + EXPECT_FALSE(muted); + } + receiver_->GetDecodingCallStatistics(&stats); + EXPECT_EQ(kNumNormalCalls + kNumPlc + kNumPlcCng, stats.calls_to_neteq); + EXPECT_EQ(0, stats.calls_to_silence_generator); + EXPECT_EQ(kNumNormalCalls, stats.decoded_normal); + EXPECT_EQ(0, stats.decoded_cng); + EXPECT_EQ(kNumPlc, stats.decoded_neteq_plc); + EXPECT_EQ(kNumPlcCng, stats.decoded_plc_cng); + EXPECT_EQ(0, stats.decoded_muted_output); + // TODO(henrik.lundin) Add a test with muted state enabled. +} + +} // namespace acm2 + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.cc b/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.cc new file mode 100644 index 0000000000..13709dbbee --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.cc @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_remixing.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +void DownMixFrame(const AudioFrame& input, rtc::ArrayView output) { + RTC_DCHECK_EQ(input.num_channels_, 2); + RTC_DCHECK_EQ(output.size(), input.samples_per_channel_); + + if (input.muted()) { + std::fill(output.begin(), output.begin() + input.samples_per_channel_, 0); + } else { + const int16_t* const input_data = input.data(); + for (size_t n = 0; n < input.samples_per_channel_; ++n) { + output[n] = rtc::dchecked_cast( + (int32_t{input_data[2 * n]} + int32_t{input_data[2 * n + 1]}) >> 1); + } + } +} + +void ReMixFrame(const AudioFrame& input, + size_t num_output_channels, + std::vector* output) { + const size_t output_size = num_output_channels * input.samples_per_channel_; + RTC_DCHECK(!(input.num_channels_ == 0 && num_output_channels > 0 && + input.samples_per_channel_ > 0)); + + if (output->size() != output_size) { + output->resize(output_size); + } + + // For muted frames, fill the frame with zeros. + if (input.muted()) { + std::fill(output->begin(), output->end(), 0); + return; + } + + // Ensure that the special case of zero input channels is handled correctly + // (zero samples per channel is already handled correctly in the code below). + if (input.num_channels_ == 0) { + return; + } + + const int16_t* const input_data = input.data(); + size_t out_index = 0; + + // When upmixing is needed and the input is mono copy the left channel + // into the left and right channels, and set any remaining channels to zero. + if (input.num_channels_ == 1 && input.num_channels_ < num_output_channels) { + for (size_t k = 0; k < input.samples_per_channel_; ++k) { + (*output)[out_index++] = input_data[k]; + (*output)[out_index++] = input_data[k]; + for (size_t j = 2; j < num_output_channels; ++j) { + (*output)[out_index++] = 0; + } + RTC_DCHECK_EQ(out_index, (k + 1) * num_output_channels); + } + RTC_DCHECK_EQ(out_index, input.samples_per_channel_ * num_output_channels); + return; + } + + size_t in_index = 0; + + // When upmixing is needed and the output is surround, copy the available + // channels directly, and set the remaining channels to zero. + if (input.num_channels_ < num_output_channels) { + for (size_t k = 0; k < input.samples_per_channel_; ++k) { + for (size_t j = 0; j < input.num_channels_; ++j) { + (*output)[out_index++] = input_data[in_index++]; + } + for (size_t j = input.num_channels_; j < num_output_channels; ++j) { + (*output)[out_index++] = 0; + } + RTC_DCHECK_EQ(in_index, (k + 1) * input.num_channels_); + RTC_DCHECK_EQ(out_index, (k + 1) * num_output_channels); + } + RTC_DCHECK_EQ(in_index, input.samples_per_channel_ * input.num_channels_); + RTC_DCHECK_EQ(out_index, input.samples_per_channel_ * num_output_channels); + + return; + } + + // When downmixing is needed, and the input is stereo, average the channels. + if (input.num_channels_ == 2) { + for (size_t n = 0; n < input.samples_per_channel_; ++n) { + (*output)[n] = rtc::dchecked_cast( + (int32_t{input_data[2 * n]} + int32_t{input_data[2 * n + 1]}) >> 1); + } + return; + } + + // When downmixing is needed, and the input is multichannel, drop the surplus + // channels. + const size_t num_channels_to_drop = input.num_channels_ - num_output_channels; + for (size_t k = 0; k < input.samples_per_channel_; ++k) { + for (size_t j = 0; j < num_output_channels; ++j) { + (*output)[out_index++] = input_data[in_index++]; + } + in_index += num_channels_to_drop; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.h b/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.h new file mode 100644 index 0000000000..661569b033 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_REMIXING_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_REMIXING_H_ + +#include + +#include "api/audio/audio_frame.h" + +namespace webrtc { + +// Stereo-to-mono downmixing. The length of the output must equal to the number +// of samples per channel in the input. +void DownMixFrame(const AudioFrame& input, rtc::ArrayView output); + +// Remixes the interleaved input frame to an interleaved output data vector. The +// remixed data replaces the data in the output vector which is resized if +// needed. The remixing supports any combination of input and output channels, +// as well as any number of samples per channel. +void ReMixFrame(const AudioFrame& input, + size_t num_output_channels, + std::vector* output); + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_ACM_REMIXING_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing_unittest.cc b/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing_unittest.cc new file mode 100644 index 0000000000..a1a816f727 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing_unittest.cc @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_remixing.h" + +#include + +#include "api/audio/audio_frame.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +using ::testing::AllOf; +using ::testing::Each; +using ::testing::ElementsAreArray; +using ::testing::SizeIs; + +namespace webrtc { + +TEST(AcmRemixing, DownMixFrame) { + std::vector out(480, 0); + AudioFrame in; + in.num_channels_ = 2; + in.samples_per_channel_ = 480; + + int16_t* const in_data = in.mutable_data(); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + in_data[2 * k] = 2; + in_data[2 * k + 1] = 0; + } + + DownMixFrame(in, out); + + EXPECT_THAT(out, AllOf(SizeIs(480), Each(1))); +} + +TEST(AcmRemixing, DownMixMutedFrame) { + std::vector out(480, 0); + AudioFrame in; + in.num_channels_ = 2; + in.samples_per_channel_ = 480; + + int16_t* const in_data = in.mutable_data(); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + in_data[2 * k] = 2; + in_data[2 * k + 1] = 0; + } + + in.Mute(); + + DownMixFrame(in, out); + + EXPECT_THAT(out, AllOf(SizeIs(480), Each(0))); +} + +TEST(AcmRemixing, RemixMutedStereoFrameTo6Channels) { + std::vector out(480, 0); + AudioFrame in; + in.num_channels_ = 2; + in.samples_per_channel_ = 480; + + int16_t* const in_data = in.mutable_data(); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + in_data[2 * k] = 1; + in_data[2 * k + 1] = 2; + } + in.Mute(); + + ReMixFrame(in, 6, &out); + EXPECT_EQ(6 * 480u, out.size()); + + EXPECT_THAT(out, AllOf(SizeIs(in.samples_per_channel_ * 6), Each(0))); +} + +TEST(AcmRemixing, RemixStereoFrameTo6Channels) { + std::vector out(480, 0); + AudioFrame in; + in.num_channels_ = 2; + in.samples_per_channel_ = 480; + + int16_t* const in_data = in.mutable_data(); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + in_data[2 * k] = 1; + in_data[2 * k + 1] = 2; + } + + ReMixFrame(in, 6, &out); + EXPECT_EQ(6 * 480u, out.size()); + + std::vector expected_output(in.samples_per_channel_ * 6); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + expected_output[6 * k] = 1; + expected_output[6 * k + 1] = 2; + } + + EXPECT_THAT(out, ElementsAreArray(expected_output)); +} + +TEST(AcmRemixing, RemixMonoFrameTo6Channels) { + std::vector out(480, 0); + AudioFrame in; + in.num_channels_ = 1; + in.samples_per_channel_ = 480; + + int16_t* const in_data = in.mutable_data(); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + in_data[k] = 1; + } + + ReMixFrame(in, 6, &out); + EXPECT_EQ(6 * 480u, out.size()); + + std::vector expected_output(in.samples_per_channel_ * 6, 0); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + expected_output[6 * k] = 1; + expected_output[6 * k + 1] = 1; + } + + EXPECT_THAT(out, ElementsAreArray(expected_output)); +} + +TEST(AcmRemixing, RemixStereoFrameToMono) { + std::vector out(480, 0); + AudioFrame in; + in.num_channels_ = 2; + in.samples_per_channel_ = 480; + + int16_t* const in_data = in.mutable_data(); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + in_data[2 * k] = 2; + in_data[2 * k + 1] = 0; + } + + ReMixFrame(in, 1, &out); + EXPECT_EQ(480u, out.size()); + + EXPECT_THAT(out, AllOf(SizeIs(in.samples_per_channel_), Each(1))); +} + +TEST(AcmRemixing, RemixMonoFrameToStereo) { + std::vector out(480, 0); + AudioFrame in; + in.num_channels_ = 1; + in.samples_per_channel_ = 480; + + int16_t* const in_data = in.mutable_data(); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + in_data[k] = 1; + } + + ReMixFrame(in, 2, &out); + EXPECT_EQ(960u, out.size()); + + EXPECT_THAT(out, AllOf(SizeIs(2 * in.samples_per_channel_), Each(1))); +} + +TEST(AcmRemixing, Remix3ChannelFrameToStereo) { + std::vector out(480, 0); + AudioFrame in; + in.num_channels_ = 3; + in.samples_per_channel_ = 480; + + int16_t* const in_data = in.mutable_data(); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + for (size_t j = 0; j < 3; ++j) { + in_data[3 * k + j] = j; + } + } + + ReMixFrame(in, 2, &out); + EXPECT_EQ(2 * 480u, out.size()); + + std::vector expected_output(in.samples_per_channel_ * 2); + for (size_t k = 0; k < in.samples_per_channel_; ++k) { + for (size_t j = 0; j < 2; ++j) { + expected_output[2 * k + j] = static_cast(j); + } + } + + EXPECT_THAT(out, ElementsAreArray(expected_output)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.cc b/third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.cc new file mode 100644 index 0000000000..e307c6ca57 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_resampler.h" + +#include + +#include "rtc_base/logging.h" + +namespace webrtc { +namespace acm2 { + +ACMResampler::ACMResampler() {} + +ACMResampler::~ACMResampler() {} + +int ACMResampler::Resample10Msec(const int16_t* in_audio, + int in_freq_hz, + int out_freq_hz, + size_t num_audio_channels, + size_t out_capacity_samples, + int16_t* out_audio) { + size_t in_length = in_freq_hz * num_audio_channels / 100; + if (in_freq_hz == out_freq_hz) { + if (out_capacity_samples < in_length) { + RTC_DCHECK_NOTREACHED(); + return -1; + } + memcpy(out_audio, in_audio, in_length * sizeof(int16_t)); + return static_cast(in_length / num_audio_channels); + } + + if (resampler_.InitializeIfNeeded(in_freq_hz, out_freq_hz, + num_audio_channels) != 0) { + RTC_LOG(LS_ERROR) << "InitializeIfNeeded(" << in_freq_hz << ", " + << out_freq_hz << ", " << num_audio_channels + << ") failed."; + return -1; + } + + int out_length = + resampler_.Resample(in_audio, in_length, out_audio, out_capacity_samples); + if (out_length == -1) { + RTC_LOG(LS_ERROR) << "Resample(" << in_audio << ", " << in_length << ", " + << out_audio << ", " << out_capacity_samples + << ") failed."; + return -1; + } + + return static_cast(out_length / num_audio_channels); +} + +} // namespace acm2 +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.h b/third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.h new file mode 100644 index 0000000000..96ba93a762 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_RESAMPLER_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_RESAMPLER_H_ + +#include +#include + +#include "common_audio/resampler/include/push_resampler.h" + +namespace webrtc { +namespace acm2 { + +class ACMResampler { + public: + ACMResampler(); + ~ACMResampler(); + + int Resample10Msec(const int16_t* in_audio, + int in_freq_hz, + int out_freq_hz, + size_t num_audio_channels, + size_t out_capacity_samples, + int16_t* out_audio); + + private: + PushResampler resampler_; +}; + +} // namespace acm2 +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_ACM_RESAMPLER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_send_test.cc b/third_party/libwebrtc/modules/audio_coding/acm2/acm_send_test.cc new file mode 100644 index 0000000000..3e65f94b0d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_send_test.cc @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/acm_send_test.h" + +#include +#include + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "rtc_base/checks.h" +#include "rtc_base/string_encode.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +AcmSendTestOldApi::AcmSendTestOldApi(InputAudioFile* audio_source, + int source_rate_hz, + int test_duration_ms) + : clock_(0), + acm_(webrtc::AudioCodingModule::Create([this] { + AudioCodingModule::Config config; + config.clock = &clock_; + config.decoder_factory = CreateBuiltinAudioDecoderFactory(); + return config; + }())), + audio_source_(audio_source), + source_rate_hz_(source_rate_hz), + input_block_size_samples_( + static_cast(source_rate_hz_ * kBlockSizeMs / 1000)), + codec_registered_(false), + test_duration_ms_(test_duration_ms), + frame_type_(AudioFrameType::kAudioFrameSpeech), + payload_type_(0), + timestamp_(0), + sequence_number_(0) { + input_frame_.sample_rate_hz_ = source_rate_hz_; + input_frame_.num_channels_ = 1; + input_frame_.samples_per_channel_ = input_block_size_samples_; + RTC_DCHECK_LE(input_block_size_samples_ * input_frame_.num_channels_, + AudioFrame::kMaxDataSizeSamples); + acm_->RegisterTransportCallback(this); +} + +AcmSendTestOldApi::~AcmSendTestOldApi() = default; + +bool AcmSendTestOldApi::RegisterCodec(absl::string_view payload_name, + int clockrate_hz, + int num_channels, + int payload_type, + int frame_size_samples) { + SdpAudioFormat format(payload_name, clockrate_hz, num_channels); + if (absl::EqualsIgnoreCase(payload_name, "g722")) { + RTC_CHECK_EQ(16000, clockrate_hz); + format.clockrate_hz = 8000; + } else if (absl::EqualsIgnoreCase(payload_name, "opus")) { + RTC_CHECK(num_channels == 1 || num_channels == 2); + if (num_channels == 2) { + format.parameters["stereo"] = "1"; + } + format.num_channels = 2; + } + format.parameters["ptime"] = rtc::ToString(rtc::CheckedDivExact( + frame_size_samples, rtc::CheckedDivExact(clockrate_hz, 1000))); + auto factory = CreateBuiltinAudioEncoderFactory(); + acm_->SetEncoder( + factory->MakeAudioEncoder(payload_type, format, absl::nullopt)); + codec_registered_ = true; + input_frame_.num_channels_ = num_channels; + RTC_DCHECK_LE(input_block_size_samples_ * input_frame_.num_channels_, + AudioFrame::kMaxDataSizeSamples); + return codec_registered_; +} + +void AcmSendTestOldApi::RegisterExternalCodec( + std::unique_ptr external_speech_encoder) { + input_frame_.num_channels_ = external_speech_encoder->NumChannels(); + acm_->SetEncoder(std::move(external_speech_encoder)); + RTC_DCHECK_LE(input_block_size_samples_ * input_frame_.num_channels_, + AudioFrame::kMaxDataSizeSamples); + codec_registered_ = true; +} + +std::unique_ptr AcmSendTestOldApi::NextPacket() { + RTC_DCHECK(codec_registered_); + if (filter_.test(static_cast(payload_type_))) { + // This payload type should be filtered out. Since the payload type is the + // same throughout the whole test run, no packet at all will be delivered. + // We can just as well signal that the test is over by returning NULL. + return nullptr; + } + // Insert audio and process until one packet is produced. + while (clock_.TimeInMilliseconds() < test_duration_ms_) { + clock_.AdvanceTimeMilliseconds(kBlockSizeMs); + RTC_CHECK(audio_source_->Read( + input_block_size_samples_ * input_frame_.num_channels_, + input_frame_.mutable_data())); + data_to_send_ = false; + RTC_CHECK_GE(acm_->Add10MsData(input_frame_), 0); + input_frame_.timestamp_ += static_cast(input_block_size_samples_); + if (data_to_send_) { + // Encoded packet received. + return CreatePacket(); + } + } + // Test ended. + return nullptr; +} + +// This method receives the callback from ACM when a new packet is produced. +int32_t AcmSendTestOldApi::SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + int64_t absolute_capture_timestamp_ms) { + // Store the packet locally. + frame_type_ = frame_type; + payload_type_ = payload_type; + timestamp_ = timestamp; + last_payload_vec_.assign(payload_data, payload_data + payload_len_bytes); + RTC_DCHECK_EQ(last_payload_vec_.size(), payload_len_bytes); + data_to_send_ = true; + return 0; +} + +std::unique_ptr AcmSendTestOldApi::CreatePacket() { + const size_t kRtpHeaderSize = 12; + rtc::CopyOnWriteBuffer packet_buffer(last_payload_vec_.size() + + kRtpHeaderSize); + uint8_t* packet_memory = packet_buffer.MutableData(); + // Populate the header bytes. + packet_memory[0] = 0x80; + packet_memory[1] = static_cast(payload_type_); + packet_memory[2] = (sequence_number_ >> 8) & 0xFF; + packet_memory[3] = (sequence_number_)&0xFF; + packet_memory[4] = (timestamp_ >> 24) & 0xFF; + packet_memory[5] = (timestamp_ >> 16) & 0xFF; + packet_memory[6] = (timestamp_ >> 8) & 0xFF; + packet_memory[7] = timestamp_ & 0xFF; + // Set SSRC to 0x12345678. + packet_memory[8] = 0x12; + packet_memory[9] = 0x34; + packet_memory[10] = 0x56; + packet_memory[11] = 0x78; + + ++sequence_number_; + + // Copy the payload data. + memcpy(packet_memory + kRtpHeaderSize, &last_payload_vec_[0], + last_payload_vec_.size()); + auto packet = std::make_unique(std::move(packet_buffer), + clock_.TimeInMilliseconds()); + RTC_DCHECK(packet); + RTC_DCHECK(packet->valid_header()); + return packet; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/acm_send_test.h b/third_party/libwebrtc/modules/audio_coding/acm2/acm_send_test.h new file mode 100644 index 0000000000..0bd24705fd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/acm_send_test.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_ACM_SEND_TEST_H_ +#define MODULES_AUDIO_CODING_ACM2_ACM_SEND_TEST_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "api/audio/audio_frame.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +class AudioEncoder; + +namespace test { +class InputAudioFile; +class Packet; + +class AcmSendTestOldApi : public AudioPacketizationCallback, + public PacketSource { + public: + AcmSendTestOldApi(InputAudioFile* audio_source, + int source_rate_hz, + int test_duration_ms); + ~AcmSendTestOldApi() override; + + AcmSendTestOldApi(const AcmSendTestOldApi&) = delete; + AcmSendTestOldApi& operator=(const AcmSendTestOldApi&) = delete; + + // Registers the send codec. Returns true on success, false otherwise. + bool RegisterCodec(absl::string_view payload_name, + int sampling_freq_hz, + int channels, + int payload_type, + int frame_size_samples); + + // Registers an external send codec. + void RegisterExternalCodec( + std::unique_ptr external_speech_encoder); + + // Inherited from PacketSource. + std::unique_ptr NextPacket() override; + + // Inherited from AudioPacketizationCallback. + int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + int64_t absolute_capture_timestamp_ms) override; + + AudioCodingModule* acm() { return acm_.get(); } + + private: + static const int kBlockSizeMs = 10; + + // Creates a Packet object from the last packet produced by ACM (and received + // through the SendData method as a callback). + std::unique_ptr CreatePacket(); + + SimulatedClock clock_; + std::unique_ptr acm_; + InputAudioFile* audio_source_; + int source_rate_hz_; + const size_t input_block_size_samples_; + AudioFrame input_frame_; + bool codec_registered_; + int test_duration_ms_; + // The following member variables are set whenever SendData() is called. + AudioFrameType frame_type_; + int payload_type_; + uint32_t timestamp_; + uint16_t sequence_number_; + std::vector last_payload_vec_; + bool data_to_send_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_ACM2_ACM_SEND_TEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module.cc b/third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module.cc new file mode 100644 index 0000000000..2c186273b6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module.cc @@ -0,0 +1,637 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/include/audio_coding_module.h" + +#include +#include + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "modules/audio_coding/acm2/acm_receiver.h" +#include "modules/audio_coding/acm2/acm_remixing.h" +#include "modules/audio_coding/acm2/acm_resampler.h" +#include "modules/include/module_common_types.h" +#include "modules/include/module_common_types_public.h" +#include "rtc_base/buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +// Initial size for the buffer in InputBuffer. This matches 6 channels of 10 ms +// 48 kHz data. +constexpr size_t kInitialInputDataBufferSize = 6 * 480; + +constexpr int32_t kMaxInputSampleRateHz = 192000; + +class AudioCodingModuleImpl final : public AudioCodingModule { + public: + explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config); + ~AudioCodingModuleImpl() override; + + ///////////////////////////////////////// + // Sender + // + + void ModifyEncoder(rtc::FunctionView*)> + modifier) override; + + // Register a transport callback which will be + // called to deliver the encoded buffers. + int RegisterTransportCallback(AudioPacketizationCallback* transport) override; + + // Add 10 ms of raw (PCM) audio data to the encoder. + int Add10MsData(const AudioFrame& audio_frame) override; + + ///////////////////////////////////////// + // (FEC) Forward Error Correction (codec internal) + // + + // Set target packet loss rate + int SetPacketLossRate(int loss_rate) override; + + ///////////////////////////////////////// + // Receiver + // + + // Initialize receiver, resets codec database etc. + int InitializeReceiver() override; + + void SetReceiveCodecs(const std::map& codecs) override; + + // Incoming packet from network parsed and ready for decode. + int IncomingPacket(const uint8_t* incoming_payload, + const size_t payload_length, + const RTPHeader& rtp_info) override; + + // Get 10 milliseconds of raw audio data to play out, and + // automatic resample to the requested frequency if > 0. + int PlayoutData10Ms(int desired_freq_hz, + AudioFrame* audio_frame, + bool* muted) override; + + ///////////////////////////////////////// + // Statistics + // + + int GetNetworkStatistics(NetworkStatistics* statistics) override; + + ANAStats GetANAStats() const override; + + int GetTargetBitrate() const override; + + private: + struct InputData { + InputData() : buffer(kInitialInputDataBufferSize) {} + uint32_t input_timestamp; + const int16_t* audio; + size_t length_per_channel; + size_t audio_channel; + // If a re-mix is required (up or down), this buffer will store a re-mixed + // version of the input. + std::vector buffer; + }; + + InputData input_data_ RTC_GUARDED_BY(acm_mutex_); + + // This member class writes values to the named UMA histogram, but only if + // the value has changed since the last time (and always for the first call). + class ChangeLogger { + public: + explicit ChangeLogger(absl::string_view histogram_name) + : histogram_name_(histogram_name) {} + // Logs the new value if it is different from the last logged value, or if + // this is the first call. + void MaybeLog(int value); + + private: + int last_value_ = 0; + int first_time_ = true; + const std::string histogram_name_; + }; + + int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data) + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_mutex_); + + // TODO(bugs.webrtc.org/10739): change `absolute_capture_timestamp_ms` to + // int64_t when it always receives a valid value. + int Encode(const InputData& input_data, + absl::optional absolute_capture_timestamp_ms) + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_mutex_); + + int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_mutex_); + + bool HaveValidEncoder(absl::string_view caller_name) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_mutex_); + + // Preprocessing of input audio, including resampling and down-mixing if + // required, before pushing audio into encoder's buffer. + // + // in_frame: input audio-frame + // ptr_out: pointer to output audio_frame. If no preprocessing is required + // `ptr_out` will be pointing to `in_frame`, otherwise pointing to + // `preprocess_frame_`. + // + // Return value: + // -1: if encountering an error. + // 0: otherwise. + int PreprocessToAddData(const AudioFrame& in_frame, + const AudioFrame** ptr_out) + RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_mutex_); + + // Change required states after starting to receive the codec corresponding + // to `index`. + int UpdateUponReceivingCodec(int index); + + mutable Mutex acm_mutex_; + rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_mutex_); + uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_mutex_); + uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_mutex_); + acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_mutex_); + acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock. + ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_mutex_); + + // Current encoder stack, provided by a call to RegisterEncoder. + std::unique_ptr encoder_stack_ RTC_GUARDED_BY(acm_mutex_); + + // This is to keep track of CN instances where we can send DTMFs. + uint8_t previous_pltype_ RTC_GUARDED_BY(acm_mutex_); + + bool receiver_initialized_ RTC_GUARDED_BY(acm_mutex_); + + AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_mutex_); + bool first_10ms_data_ RTC_GUARDED_BY(acm_mutex_); + + bool first_frame_ RTC_GUARDED_BY(acm_mutex_); + uint32_t last_timestamp_ RTC_GUARDED_BY(acm_mutex_); + uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_mutex_); + + Mutex callback_mutex_; + AudioPacketizationCallback* packetization_callback_ + RTC_GUARDED_BY(callback_mutex_); + + int codec_histogram_bins_log_[static_cast( + AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)]; + int number_of_consecutive_empty_packets_; +}; + +// Adds a codec usage sample to the histogram. +void UpdateCodecTypeHistogram(size_t codec_type) { + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.Encoder.CodecType", static_cast(codec_type), + static_cast( + webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)); +} + +void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) { + if (value != last_value_ || first_time_) { + first_time_ = false; + last_value_ = value; + RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value); + } +} + +AudioCodingModuleImpl::AudioCodingModuleImpl( + const AudioCodingModule::Config& config) + : expected_codec_ts_(0xD87F3F9F), + expected_in_ts_(0xD87F3F9F), + receiver_(config), + bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"), + encoder_stack_(nullptr), + previous_pltype_(255), + receiver_initialized_(false), + first_10ms_data_(false), + first_frame_(true), + packetization_callback_(NULL), + codec_histogram_bins_log_(), + number_of_consecutive_empty_packets_(0) { + if (InitializeReceiverSafe() < 0) { + RTC_LOG(LS_ERROR) << "Cannot initialize receiver"; + } + RTC_LOG(LS_INFO) << "Created"; +} + +AudioCodingModuleImpl::~AudioCodingModuleImpl() = default; + +int32_t AudioCodingModuleImpl::Encode( + const InputData& input_data, + absl::optional absolute_capture_timestamp_ms) { + // TODO(bugs.webrtc.org/10739): add dcheck that + // `audio_frame.absolute_capture_timestamp_ms()` always has a value. + AudioEncoder::EncodedInfo encoded_info; + uint8_t previous_pltype; + + // Check if there is an encoder before. + if (!HaveValidEncoder("Process")) + return -1; + + if (!first_frame_) { + RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_)) + << "Time should not move backwards"; + } + + // Scale the timestamp to the codec's RTP timestamp rate. + uint32_t rtp_timestamp = + first_frame_ + ? input_data.input_timestamp + : last_rtp_timestamp_ + + rtc::dchecked_cast(rtc::CheckedDivExact( + int64_t{input_data.input_timestamp - last_timestamp_} * + encoder_stack_->RtpTimestampRateHz(), + int64_t{encoder_stack_->SampleRateHz()})); + + last_timestamp_ = input_data.input_timestamp; + last_rtp_timestamp_ = rtp_timestamp; + first_frame_ = false; + + // Clear the buffer before reuse - encoded data will get appended. + encode_buffer_.Clear(); + encoded_info = encoder_stack_->Encode( + rtp_timestamp, + rtc::ArrayView( + input_data.audio, + input_data.audio_channel * input_data.length_per_channel), + &encode_buffer_); + + bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000); + if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) { + // Not enough data. + return 0; + } + previous_pltype = previous_pltype_; // Read it while we have the critsect. + + // Log codec type to histogram once every 500 packets. + if (encoded_info.encoded_bytes == 0) { + ++number_of_consecutive_empty_packets_; + } else { + size_t codec_type = static_cast(encoded_info.encoder_type); + codec_histogram_bins_log_[codec_type] += + number_of_consecutive_empty_packets_ + 1; + number_of_consecutive_empty_packets_ = 0; + if (codec_histogram_bins_log_[codec_type] >= 500) { + codec_histogram_bins_log_[codec_type] -= 500; + UpdateCodecTypeHistogram(codec_type); + } + } + + AudioFrameType frame_type; + if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) { + frame_type = AudioFrameType::kEmptyFrame; + encoded_info.payload_type = previous_pltype; + } else { + RTC_DCHECK_GT(encode_buffer_.size(), 0); + frame_type = encoded_info.speech ? AudioFrameType::kAudioFrameSpeech + : AudioFrameType::kAudioFrameCN; + } + + { + MutexLock lock(&callback_mutex_); + if (packetization_callback_) { + packetization_callback_->SendData( + frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp, + encode_buffer_.data(), encode_buffer_.size(), + absolute_capture_timestamp_ms.value_or(-1)); + } + } + previous_pltype_ = encoded_info.payload_type; + return static_cast(encode_buffer_.size()); +} + +///////////////////////////////////////// +// Sender +// + +void AudioCodingModuleImpl::ModifyEncoder( + rtc::FunctionView*)> modifier) { + MutexLock lock(&acm_mutex_); + modifier(&encoder_stack_); +} + +// Register a transport callback which will be called to deliver +// the encoded buffers. +int AudioCodingModuleImpl::RegisterTransportCallback( + AudioPacketizationCallback* transport) { + MutexLock lock(&callback_mutex_); + packetization_callback_ = transport; + return 0; +} + +// Add 10MS of raw (PCM) audio data to the encoder. +int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) { + MutexLock lock(&acm_mutex_); + int r = Add10MsDataInternal(audio_frame, &input_data_); + // TODO(bugs.webrtc.org/10739): add dcheck that + // `audio_frame.absolute_capture_timestamp_ms()` always has a value. + return r < 0 + ? r + : Encode(input_data_, audio_frame.absolute_capture_timestamp_ms()); +} + +int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame, + InputData* input_data) { + if (audio_frame.samples_per_channel_ == 0) { + RTC_DCHECK_NOTREACHED(); + RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero"; + return -1; + } + + if (audio_frame.sample_rate_hz_ > kMaxInputSampleRateHz) { + RTC_DCHECK_NOTREACHED(); + RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid"; + return -1; + } + + // If the length and frequency matches. We currently just support raw PCM. + if (static_cast(audio_frame.sample_rate_hz_ / 100) != + audio_frame.samples_per_channel_) { + RTC_LOG(LS_ERROR) + << "Cannot Add 10 ms audio, input frequency and length doesn't match"; + return -1; + } + + if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2 && + audio_frame.num_channels_ != 4 && audio_frame.num_channels_ != 6 && + audio_frame.num_channels_ != 8) { + RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels."; + return -1; + } + + // Do we have a codec registered? + if (!HaveValidEncoder("Add10MsData")) { + return -1; + } + + const AudioFrame* ptr_frame; + // Perform a resampling, also down-mix if it is required and can be + // performed before resampling (a down mix prior to resampling will take + // place if both primary and secondary encoders are mono and input is in + // stereo). + if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) { + return -1; + } + + // Check whether we need an up-mix or down-mix? + const size_t current_num_channels = encoder_stack_->NumChannels(); + const bool same_num_channels = + ptr_frame->num_channels_ == current_num_channels; + + // TODO(yujo): Skip encode of muted frames. + input_data->input_timestamp = ptr_frame->timestamp_; + input_data->length_per_channel = ptr_frame->samples_per_channel_; + input_data->audio_channel = current_num_channels; + + if (!same_num_channels) { + // Remixes the input frame to the output data and in the process resize the + // output data if needed. + ReMixFrame(*ptr_frame, current_num_channels, &input_data->buffer); + + // For pushing data to primary, point the `ptr_audio` to correct buffer. + input_data->audio = input_data->buffer.data(); + RTC_DCHECK_GE(input_data->buffer.size(), + input_data->length_per_channel * input_data->audio_channel); + } else { + // When adding data to encoders this pointer is pointing to an audio buffer + // with correct number of channels. + input_data->audio = ptr_frame->data(); + } + + return 0; +} + +// Perform a resampling and down-mix if required. We down-mix only if +// encoder is mono and input is stereo. In case of dual-streaming, both +// encoders has to be mono for down-mix to take place. +// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing +// is required, |*ptr_out| points to `in_frame`. +// TODO(yujo): Make this more efficient for muted frames. +int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame, + const AudioFrame** ptr_out) { + const bool resample = + in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz(); + + // This variable is true if primary codec and secondary codec (if exists) + // are both mono and input is stereo. + // TODO(henrik.lundin): This condition should probably be + // in_frame.num_channels_ > encoder_stack_->NumChannels() + const bool down_mix = + in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1; + + if (!first_10ms_data_) { + expected_in_ts_ = in_frame.timestamp_; + expected_codec_ts_ = in_frame.timestamp_; + first_10ms_data_ = true; + } else if (in_frame.timestamp_ != expected_in_ts_) { + RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_ + << ", expected: " << expected_in_ts_; + expected_codec_ts_ += + (in_frame.timestamp_ - expected_in_ts_) * + static_cast( + static_cast(encoder_stack_->SampleRateHz()) / + static_cast(in_frame.sample_rate_hz_)); + expected_in_ts_ = in_frame.timestamp_; + } + + if (!down_mix && !resample) { + // No pre-processing is required. + if (expected_in_ts_ == expected_codec_ts_) { + // If we've never resampled, we can use the input frame as-is + *ptr_out = &in_frame; + } else { + // Otherwise we'll need to alter the timestamp. Since in_frame is const, + // we'll have to make a copy of it. + preprocess_frame_.CopyFrom(in_frame); + preprocess_frame_.timestamp_ = expected_codec_ts_; + *ptr_out = &preprocess_frame_; + } + + expected_in_ts_ += static_cast(in_frame.samples_per_channel_); + expected_codec_ts_ += static_cast(in_frame.samples_per_channel_); + return 0; + } + + *ptr_out = &preprocess_frame_; + preprocess_frame_.num_channels_ = in_frame.num_channels_; + preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_; + std::array audio; + const int16_t* src_ptr_audio; + if (down_mix) { + // If a resampling is required, the output of a down-mix is written into a + // local buffer, otherwise, it will be written to the output frame. + int16_t* dest_ptr_audio = + resample ? audio.data() : preprocess_frame_.mutable_data(); + RTC_DCHECK_GE(audio.size(), preprocess_frame_.samples_per_channel_); + RTC_DCHECK_GE(audio.size(), in_frame.samples_per_channel_); + DownMixFrame(in_frame, + rtc::ArrayView( + dest_ptr_audio, preprocess_frame_.samples_per_channel_)); + preprocess_frame_.num_channels_ = 1; + + // Set the input of the resampler to the down-mixed signal. + src_ptr_audio = audio.data(); + } else { + // Set the input of the resampler to the original data. + src_ptr_audio = in_frame.data(); + } + + preprocess_frame_.timestamp_ = expected_codec_ts_; + preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_; + // If it is required, we have to do a resampling. + if (resample) { + // The result of the resampler is written to output frame. + int16_t* dest_ptr_audio = preprocess_frame_.mutable_data(); + + int samples_per_channel = resampler_.Resample10Msec( + src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(), + preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples, + dest_ptr_audio); + + if (samples_per_channel < 0) { + RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed"; + return -1; + } + preprocess_frame_.samples_per_channel_ = + static_cast(samples_per_channel); + preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz(); + } + + expected_codec_ts_ += + static_cast(preprocess_frame_.samples_per_channel_); + expected_in_ts_ += static_cast(in_frame.samples_per_channel_); + + return 0; +} + +///////////////////////////////////////// +// (FEC) Forward Error Correction (codec internal) +// + +int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) { + MutexLock lock(&acm_mutex_); + if (HaveValidEncoder("SetPacketLossRate")) { + encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0); + } + return 0; +} + +///////////////////////////////////////// +// Receiver +// + +int AudioCodingModuleImpl::InitializeReceiver() { + MutexLock lock(&acm_mutex_); + return InitializeReceiverSafe(); +} + +// Initialize receiver, resets codec database etc. +int AudioCodingModuleImpl::InitializeReceiverSafe() { + // If the receiver is already initialized then we want to destroy any + // existing decoders. After a call to this function, we should have a clean + // start-up. + if (receiver_initialized_) + receiver_.RemoveAllCodecs(); + receiver_.FlushBuffers(); + + receiver_initialized_ = true; + return 0; +} + +void AudioCodingModuleImpl::SetReceiveCodecs( + const std::map& codecs) { + MutexLock lock(&acm_mutex_); + receiver_.SetCodecs(codecs); +} + +// Incoming packet from network parsed and ready for decode. +int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload, + const size_t payload_length, + const RTPHeader& rtp_header) { + RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr); + return receiver_.InsertPacket( + rtp_header, + rtc::ArrayView(incoming_payload, payload_length)); +} + +// Get 10 milliseconds of raw audio data to play out. +// Automatic resample to the requested frequency. +int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz, + AudioFrame* audio_frame, + bool* muted) { + // GetAudio always returns 10 ms, at the requested sample rate. + if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) { + RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed"; + return -1; + } + return 0; +} + +///////////////////////////////////////// +// Statistics +// + +// TODO(turajs) change the return value to void. Also change the corresponding +// NetEq function. +int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) { + receiver_.GetNetworkStatistics(statistics); + return 0; +} + +bool AudioCodingModuleImpl::HaveValidEncoder( + absl::string_view caller_name) const { + if (!encoder_stack_) { + RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered."; + return false; + } + return true; +} + +ANAStats AudioCodingModuleImpl::GetANAStats() const { + MutexLock lock(&acm_mutex_); + if (encoder_stack_) + return encoder_stack_->GetANAStats(); + // If no encoder is set, return default stats. + return ANAStats(); +} + +int AudioCodingModuleImpl::GetTargetBitrate() const { + MutexLock lock(&acm_mutex_); + if (!encoder_stack_) { + return -1; + } + return encoder_stack_->GetTargetBitrate(); +} + +} // namespace + +AudioCodingModule::Config::Config( + rtc::scoped_refptr decoder_factory) + : neteq_config(), + clock(Clock::GetRealTimeClockRaw()), + decoder_factory(decoder_factory) { + // Post-decode VAD is disabled by default in NetEq, however, Audio + // Conference Mixer relies on VAD decisions and fails without them. + neteq_config.enable_post_decode_vad = true; +} + +AudioCodingModule::Config::Config(const Config&) = default; +AudioCodingModule::Config::~Config() = default; + +AudioCodingModule* AudioCodingModule::Create(const Config& config) { + return new AudioCodingModuleImpl(config); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc b/third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc new file mode 100644 index 0000000000..f1eb81c015 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc @@ -0,0 +1,1278 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/include/audio_coding_module.h" + +#include +#include + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/audio_codecs/opus/audio_decoder_multi_channel_opus.h" +#include "api/audio_codecs/opus/audio_decoder_opus.h" +#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/acm2/acm_receive_test.h" +#include "modules/audio_coding/acm2/acm_send_test.h" +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" +#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h" +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/neteq/tools/audio_checksum.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "modules/audio_coding/neteq/tools/constant_pcm_packet_source.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/output_audio_file.h" +#include "modules/audio_coding/neteq/tools/output_wav_file.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "rtc_base/event.h" +#include "rtc_base/message_digest.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/arch.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "system_wrappers/include/cpu_features_wrapper.h" +#include "system_wrappers/include/sleep.h" +#include "test/audio_decoder_proxy_factory.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder.h" +#include "test/mock_audio_encoder.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/rtc_expect_death.h" + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::Invoke; + +namespace webrtc { + +namespace { +const int kSampleRateHz = 16000; +const int kNumSamples10ms = kSampleRateHz / 100; +const int kFrameSizeMs = 10; // Multiple of 10. +const int kFrameSizeSamples = kFrameSizeMs / 10 * kNumSamples10ms; +const int kPayloadSizeBytes = kFrameSizeSamples * sizeof(int16_t); +const uint8_t kPayloadType = 111; +} // namespace + +class RtpData { + public: + RtpData(int samples_per_packet, uint8_t payload_type) + : samples_per_packet_(samples_per_packet), payload_type_(payload_type) {} + + virtual ~RtpData() {} + + void Populate(RTPHeader* rtp_header) { + rtp_header->sequenceNumber = 0xABCD; + rtp_header->timestamp = 0xABCDEF01; + rtp_header->payloadType = payload_type_; + rtp_header->markerBit = false; + rtp_header->ssrc = 0x1234; + rtp_header->numCSRCs = 0; + + rtp_header->payload_type_frequency = kSampleRateHz; + } + + void Forward(RTPHeader* rtp_header) { + ++rtp_header->sequenceNumber; + rtp_header->timestamp += samples_per_packet_; + } + + private: + int samples_per_packet_; + uint8_t payload_type_; +}; + +class PacketizationCallbackStubOldApi : public AudioPacketizationCallback { + public: + PacketizationCallbackStubOldApi() + : num_calls_(0), + last_frame_type_(AudioFrameType::kEmptyFrame), + last_payload_type_(-1), + last_timestamp_(0) {} + + int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + int64_t absolute_capture_timestamp_ms) override { + MutexLock lock(&mutex_); + ++num_calls_; + last_frame_type_ = frame_type; + last_payload_type_ = payload_type; + last_timestamp_ = timestamp; + last_payload_vec_.assign(payload_data, payload_data + payload_len_bytes); + return 0; + } + + int num_calls() const { + MutexLock lock(&mutex_); + return num_calls_; + } + + int last_payload_len_bytes() const { + MutexLock lock(&mutex_); + return rtc::checked_cast(last_payload_vec_.size()); + } + + AudioFrameType last_frame_type() const { + MutexLock lock(&mutex_); + return last_frame_type_; + } + + int last_payload_type() const { + MutexLock lock(&mutex_); + return last_payload_type_; + } + + uint32_t last_timestamp() const { + MutexLock lock(&mutex_); + return last_timestamp_; + } + + void SwapBuffers(std::vector* payload) { + MutexLock lock(&mutex_); + last_payload_vec_.swap(*payload); + } + + private: + int num_calls_ RTC_GUARDED_BY(mutex_); + AudioFrameType last_frame_type_ RTC_GUARDED_BY(mutex_); + int last_payload_type_ RTC_GUARDED_BY(mutex_); + uint32_t last_timestamp_ RTC_GUARDED_BY(mutex_); + std::vector last_payload_vec_ RTC_GUARDED_BY(mutex_); + mutable Mutex mutex_; +}; + +class AudioCodingModuleTestOldApi : public ::testing::Test { + protected: + AudioCodingModuleTestOldApi() + : rtp_utility_(new RtpData(kFrameSizeSamples, kPayloadType)), + clock_(Clock::GetRealTimeClock()) {} + + ~AudioCodingModuleTestOldApi() {} + + void TearDown() {} + + void SetUp() { + acm_.reset(AudioCodingModule::Create([this] { + AudioCodingModule::Config config; + config.clock = clock_; + config.decoder_factory = CreateBuiltinAudioDecoderFactory(); + return config; + }())); + + rtp_utility_->Populate(&rtp_header_); + + input_frame_.sample_rate_hz_ = kSampleRateHz; + input_frame_.num_channels_ = 1; + input_frame_.samples_per_channel_ = kSampleRateHz * 10 / 1000; // 10 ms. + static_assert(kSampleRateHz * 10 / 1000 <= AudioFrame::kMaxDataSizeSamples, + "audio frame too small"); + input_frame_.Mute(); + + ASSERT_EQ(0, acm_->RegisterTransportCallback(&packet_cb_)); + + SetUpL16Codec(); + } + + // Set up L16 codec. + virtual void SetUpL16Codec() { + audio_format_ = SdpAudioFormat("L16", kSampleRateHz, 1); + pac_size_ = 160; + } + + virtual void RegisterCodec() { + acm_->SetReceiveCodecs({{kPayloadType, *audio_format_}}); + acm_->SetEncoder(CreateBuiltinAudioEncoderFactory()->MakeAudioEncoder( + kPayloadType, *audio_format_, absl::nullopt)); + } + + virtual void InsertPacketAndPullAudio() { + InsertPacket(); + PullAudio(); + } + + virtual void InsertPacket() { + const uint8_t kPayload[kPayloadSizeBytes] = {0}; + ASSERT_EQ(0, + acm_->IncomingPacket(kPayload, kPayloadSizeBytes, rtp_header_)); + rtp_utility_->Forward(&rtp_header_); + } + + virtual void PullAudio() { + AudioFrame audio_frame; + bool muted; + ASSERT_EQ(0, acm_->PlayoutData10Ms(-1, &audio_frame, &muted)); + ASSERT_FALSE(muted); + } + + virtual void InsertAudio() { + ASSERT_GE(acm_->Add10MsData(input_frame_), 0); + input_frame_.timestamp_ += kNumSamples10ms; + } + + virtual void VerifyEncoding() { + int last_length = packet_cb_.last_payload_len_bytes(); + EXPECT_TRUE(last_length == 2 * pac_size_ || last_length == 0) + << "Last encoded packet was " << last_length << " bytes."; + } + + virtual void InsertAudioAndVerifyEncoding() { + InsertAudio(); + VerifyEncoding(); + } + + std::unique_ptr rtp_utility_; + std::unique_ptr acm_; + PacketizationCallbackStubOldApi packet_cb_; + RTPHeader rtp_header_; + AudioFrame input_frame_; + + absl::optional audio_format_; + int pac_size_ = -1; + + Clock* clock_; +}; + +class AudioCodingModuleTestOldApiDeathTest + : public AudioCodingModuleTestOldApi {}; + +TEST_F(AudioCodingModuleTestOldApi, VerifyOutputFrame) { + AudioFrame audio_frame; + const int kSampleRateHz = 32000; + bool muted; + EXPECT_EQ(0, acm_->PlayoutData10Ms(kSampleRateHz, &audio_frame, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(0u, audio_frame.timestamp_); + EXPECT_GT(audio_frame.num_channels_, 0u); + EXPECT_EQ(static_cast(kSampleRateHz / 100), + audio_frame.samples_per_channel_); + EXPECT_EQ(kSampleRateHz, audio_frame.sample_rate_hz_); +} + +// The below test is temporarily disabled on Windows due to problems +// with clang debug builds. +// TODO(tommi): Re-enable when we've figured out what the problem is. +// http://crbug.com/615050 +#if !defined(WEBRTC_WIN) && defined(__clang__) && RTC_DCHECK_IS_ON && \ + GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST_F(AudioCodingModuleTestOldApiDeathTest, FailOnZeroDesiredFrequency) { + AudioFrame audio_frame; + bool muted; + RTC_EXPECT_DEATH(acm_->PlayoutData10Ms(0, &audio_frame, &muted), + "dst_sample_rate_hz"); +} +#endif + +// Checks that the transport callback is invoked once for each speech packet. +// Also checks that the frame type is kAudioFrameSpeech. +TEST_F(AudioCodingModuleTestOldApi, TransportCallbackIsInvokedForEachPacket) { + const int k10MsBlocksPerPacket = 3; + pac_size_ = k10MsBlocksPerPacket * kSampleRateHz / 100; + audio_format_->parameters["ptime"] = "30"; + RegisterCodec(); + const int kLoops = 10; + for (int i = 0; i < kLoops; ++i) { + EXPECT_EQ(i / k10MsBlocksPerPacket, packet_cb_.num_calls()); + if (packet_cb_.num_calls() > 0) + EXPECT_EQ(AudioFrameType::kAudioFrameSpeech, + packet_cb_.last_frame_type()); + InsertAudioAndVerifyEncoding(); + } + EXPECT_EQ(kLoops / k10MsBlocksPerPacket, packet_cb_.num_calls()); + EXPECT_EQ(AudioFrameType::kAudioFrameSpeech, packet_cb_.last_frame_type()); +} + +// Introduce this class to set different expectations on the number of encoded +// bytes. This class expects all encoded packets to be 9 bytes (matching one +// CNG SID frame) or 0 bytes. This test depends on `input_frame_` containing +// (near-)zero values. It also introduces a way to register comfort noise with +// a custom payload type. +class AudioCodingModuleTestWithComfortNoiseOldApi + : public AudioCodingModuleTestOldApi { + protected: + void RegisterCngCodec(int rtp_payload_type) { + acm_->SetReceiveCodecs({{kPayloadType, *audio_format_}, + {rtp_payload_type, {"cn", kSampleRateHz, 1}}}); + acm_->ModifyEncoder([&](std::unique_ptr* enc) { + AudioEncoderCngConfig config; + config.speech_encoder = std::move(*enc); + config.num_channels = 1; + config.payload_type = rtp_payload_type; + config.vad_mode = Vad::kVadNormal; + *enc = CreateComfortNoiseEncoder(std::move(config)); + }); + } + + void VerifyEncoding() override { + int last_length = packet_cb_.last_payload_len_bytes(); + EXPECT_TRUE(last_length == 9 || last_length == 0) + << "Last encoded packet was " << last_length << " bytes."; + } + + void DoTest(int blocks_per_packet, int cng_pt) { + const int kLoops = 40; + // This array defines the expected frame types, and when they should arrive. + // We expect a frame to arrive each time the speech encoder would have + // produced a packet, and once every 100 ms the frame should be non-empty, + // that is contain comfort noise. + const struct { + int ix; + AudioFrameType type; + } expectation[] = {{2, AudioFrameType::kAudioFrameCN}, + {5, AudioFrameType::kEmptyFrame}, + {8, AudioFrameType::kEmptyFrame}, + {11, AudioFrameType::kAudioFrameCN}, + {14, AudioFrameType::kEmptyFrame}, + {17, AudioFrameType::kEmptyFrame}, + {20, AudioFrameType::kAudioFrameCN}, + {23, AudioFrameType::kEmptyFrame}, + {26, AudioFrameType::kEmptyFrame}, + {29, AudioFrameType::kEmptyFrame}, + {32, AudioFrameType::kAudioFrameCN}, + {35, AudioFrameType::kEmptyFrame}, + {38, AudioFrameType::kEmptyFrame}}; + for (int i = 0; i < kLoops; ++i) { + int num_calls_before = packet_cb_.num_calls(); + EXPECT_EQ(i / blocks_per_packet, num_calls_before); + InsertAudioAndVerifyEncoding(); + int num_calls = packet_cb_.num_calls(); + if (num_calls == num_calls_before + 1) { + EXPECT_EQ(expectation[num_calls - 1].ix, i); + EXPECT_EQ(expectation[num_calls - 1].type, packet_cb_.last_frame_type()) + << "Wrong frame type for lap " << i; + EXPECT_EQ(cng_pt, packet_cb_.last_payload_type()); + } else { + EXPECT_EQ(num_calls, num_calls_before); + } + } + } +}; + +// Checks that the transport callback is invoked once per frame period of the +// underlying speech encoder, even when comfort noise is produced. +// Also checks that the frame type is kAudioFrameCN or kEmptyFrame. +TEST_F(AudioCodingModuleTestWithComfortNoiseOldApi, + TransportCallbackTestForComfortNoiseRegisterCngLast) { + const int k10MsBlocksPerPacket = 3; + pac_size_ = k10MsBlocksPerPacket * kSampleRateHz / 100; + audio_format_->parameters["ptime"] = "30"; + RegisterCodec(); + const int kCngPayloadType = 105; + RegisterCngCodec(kCngPayloadType); + DoTest(k10MsBlocksPerPacket, kCngPayloadType); +} + +// A multi-threaded test for ACM that uses the PCM16b 16 kHz codec. +class AudioCodingModuleMtTestOldApi : public AudioCodingModuleTestOldApi { + protected: + static const int kNumPackets = 500; + static const int kNumPullCalls = 500; + + AudioCodingModuleMtTestOldApi() + : AudioCodingModuleTestOldApi(), + send_count_(0), + insert_packet_count_(0), + pull_audio_count_(0), + next_insert_packet_time_ms_(0), + fake_clock_(new SimulatedClock(0)) { + clock_ = fake_clock_.get(); + } + + void SetUp() { + AudioCodingModuleTestOldApi::SetUp(); + RegisterCodec(); // Must be called before the threads start below. + StartThreads(); + } + + void StartThreads() { + quit_.store(false); + + const auto attributes = + rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime); + send_thread_ = rtc::PlatformThread::SpawnJoinable( + [this] { + while (!quit_.load()) { + CbSendImpl(); + } + }, + "send", attributes); + insert_packet_thread_ = rtc::PlatformThread::SpawnJoinable( + [this] { + while (!quit_.load()) { + CbInsertPacketImpl(); + } + }, + "insert_packet", attributes); + pull_audio_thread_ = rtc::PlatformThread::SpawnJoinable( + [this] { + while (!quit_.load()) { + CbPullAudioImpl(); + } + }, + "pull_audio", attributes); + } + + void TearDown() { + AudioCodingModuleTestOldApi::TearDown(); + quit_.store(true); + pull_audio_thread_.Finalize(); + send_thread_.Finalize(); + insert_packet_thread_.Finalize(); + } + + bool RunTest() { return test_complete_.Wait(TimeDelta::Minutes(10)); } + + virtual bool TestDone() { + if (packet_cb_.num_calls() > kNumPackets) { + MutexLock lock(&mutex_); + if (pull_audio_count_ > kNumPullCalls) { + // Both conditions for completion are met. End the test. + return true; + } + } + return false; + } + + // The send thread doesn't have to care about the current simulated time, + // since only the AcmReceiver is using the clock. + void CbSendImpl() { + SleepMs(1); + if (HasFatalFailure()) { + // End the test early if a fatal failure (ASSERT_*) has occurred. + test_complete_.Set(); + } + ++send_count_; + InsertAudioAndVerifyEncoding(); + if (TestDone()) { + test_complete_.Set(); + } + } + + void CbInsertPacketImpl() { + SleepMs(1); + { + MutexLock lock(&mutex_); + if (clock_->TimeInMilliseconds() < next_insert_packet_time_ms_) { + return; + } + next_insert_packet_time_ms_ += 10; + } + // Now we're not holding the crit sect when calling ACM. + ++insert_packet_count_; + InsertPacket(); + } + + void CbPullAudioImpl() { + SleepMs(1); + { + MutexLock lock(&mutex_); + // Don't let the insert thread fall behind. + if (next_insert_packet_time_ms_ < clock_->TimeInMilliseconds()) { + return; + } + ++pull_audio_count_; + } + // Now we're not holding the crit sect when calling ACM. + PullAudio(); + fake_clock_->AdvanceTimeMilliseconds(10); + } + + rtc::PlatformThread send_thread_; + rtc::PlatformThread insert_packet_thread_; + rtc::PlatformThread pull_audio_thread_; + // Used to force worker threads to stop looping. + std::atomic quit_; + + rtc::Event test_complete_; + int send_count_; + int insert_packet_count_; + int pull_audio_count_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; + int64_t next_insert_packet_time_ms_ RTC_GUARDED_BY(mutex_); + std::unique_ptr fake_clock_; +}; + +#if defined(WEBRTC_IOS) +#define MAYBE_DoTest DISABLED_DoTest +#else +#define MAYBE_DoTest DoTest +#endif +TEST_F(AudioCodingModuleMtTestOldApi, MAYBE_DoTest) { + EXPECT_TRUE(RunTest()); +} + +// Disabling all of these tests on iOS until file support has been added. +// See https://code.google.com/p/webrtc/issues/detail?id=4752 for details. +#if !defined(WEBRTC_IOS) + +// This test verifies bit exactness for the send-side of ACM. The test setup is +// a chain of three different test classes: +// +// test::AcmSendTest -> AcmSenderBitExactness -> test::AcmReceiveTest +// +// The receiver side is driving the test by requesting new packets from +// AcmSenderBitExactness::NextPacket(). This method, in turn, asks for the +// packet from test::AcmSendTest::NextPacket, which inserts audio from the +// input file until one packet is produced. (The input file loops indefinitely.) +// Before passing the packet to the receiver, this test class verifies the +// packet header and updates a payload checksum with the new payload. The +// decoded output from the receiver is also verified with a (separate) checksum. +class AcmSenderBitExactnessOldApi : public ::testing::Test, + public test::PacketSource { + protected: + static const int kTestDurationMs = 1000; + + AcmSenderBitExactnessOldApi() + : frame_size_rtp_timestamps_(0), + packet_count_(0), + payload_type_(0), + last_sequence_number_(0), + last_timestamp_(0), + payload_checksum_(rtc::MessageDigestFactory::Create(rtc::DIGEST_MD5)) {} + + // Sets up the test::AcmSendTest object. Returns true on success, otherwise + // false. + bool SetUpSender(absl::string_view input_file_name, int source_rate) { + // Note that `audio_source_` will loop forever. The test duration is set + // explicitly by `kTestDurationMs`. + audio_source_.reset(new test::InputAudioFile(input_file_name)); + send_test_.reset(new test::AcmSendTestOldApi(audio_source_.get(), + source_rate, kTestDurationMs)); + return send_test_.get() != NULL; + } + + // Registers a send codec in the test::AcmSendTest object. Returns true on + // success, false on failure. + bool RegisterSendCodec(absl::string_view payload_name, + int sampling_freq_hz, + int channels, + int payload_type, + int frame_size_samples, + int frame_size_rtp_timestamps) { + payload_type_ = payload_type; + frame_size_rtp_timestamps_ = frame_size_rtp_timestamps; + return send_test_->RegisterCodec(payload_name, sampling_freq_hz, channels, + payload_type, frame_size_samples); + } + + void RegisterExternalSendCodec( + std::unique_ptr external_speech_encoder, + int payload_type) { + payload_type_ = payload_type; + frame_size_rtp_timestamps_ = rtc::checked_cast( + external_speech_encoder->Num10MsFramesInNextPacket() * + external_speech_encoder->RtpTimestampRateHz() / 100); + send_test_->RegisterExternalCodec(std::move(external_speech_encoder)); + } + + // Runs the test. SetUpSender() and RegisterSendCodec() must have been called + // before calling this method. + void Run(absl::string_view audio_checksum_ref, + absl::string_view payload_checksum_ref, + int expected_packets, + test::AcmReceiveTestOldApi::NumOutputChannels expected_channels, + rtc::scoped_refptr decoder_factory = nullptr) { + if (!decoder_factory) { + decoder_factory = CreateBuiltinAudioDecoderFactory(); + } + // Set up the receiver used to decode the packets and verify the decoded + // output. + test::AudioChecksum audio_checksum; + const std::string output_file_name = + webrtc::test::OutputPath() + + ::testing::UnitTest::GetInstance() + ->current_test_info() + ->test_case_name() + + "_" + ::testing::UnitTest::GetInstance()->current_test_info()->name() + + "_output.wav"; + const int kOutputFreqHz = 8000; + test::OutputWavFile output_file(output_file_name, kOutputFreqHz, + expected_channels); + // Have the output audio sent both to file and to the checksum calculator. + test::AudioSinkFork output(&audio_checksum, &output_file); + test::AcmReceiveTestOldApi receive_test(this, &output, kOutputFreqHz, + expected_channels, decoder_factory); + ASSERT_NO_FATAL_FAILURE(receive_test.RegisterDefaultCodecs()); + + // This is where the actual test is executed. + receive_test.Run(); + + // Extract and verify the audio checksum. + std::string checksum_string = audio_checksum.Finish(); + ExpectChecksumEq(audio_checksum_ref, checksum_string); + + // Extract and verify the payload checksum. + rtc::Buffer checksum_result(payload_checksum_->Size()); + payload_checksum_->Finish(checksum_result.data(), checksum_result.size()); + checksum_string = rtc::hex_encode(checksum_result); + ExpectChecksumEq(payload_checksum_ref, checksum_string); + + // Verify number of packets produced. + EXPECT_EQ(expected_packets, packet_count_); + + // Delete the output file. + remove(output_file_name.c_str()); + } + + // Helper: result must be one the "|"-separated checksums. + void ExpectChecksumEq(absl::string_view ref, absl::string_view result) { + if (ref.size() == result.size()) { + // Only one checksum: clearer message. + EXPECT_EQ(ref, result); + } else { + EXPECT_NE(ref.find(result), absl::string_view::npos) + << result << " must be one of these:\n" + << ref; + } + } + + // Inherited from test::PacketSource. + std::unique_ptr NextPacket() override { + auto packet = send_test_->NextPacket(); + if (!packet) + return NULL; + + VerifyPacket(packet.get()); + // TODO(henrik.lundin) Save the packet to file as well. + + // Pass it on to the caller. The caller becomes the owner of `packet`. + return packet; + } + + // Verifies the packet. + void VerifyPacket(const test::Packet* packet) { + EXPECT_TRUE(packet->valid_header()); + // (We can check the header fields even if valid_header() is false.) + EXPECT_EQ(payload_type_, packet->header().payloadType); + if (packet_count_ > 0) { + // This is not the first packet. + uint16_t sequence_number_diff = + packet->header().sequenceNumber - last_sequence_number_; + EXPECT_EQ(1, sequence_number_diff); + uint32_t timestamp_diff = packet->header().timestamp - last_timestamp_; + EXPECT_EQ(frame_size_rtp_timestamps_, timestamp_diff); + } + ++packet_count_; + last_sequence_number_ = packet->header().sequenceNumber; + last_timestamp_ = packet->header().timestamp; + // Update the checksum. + payload_checksum_->Update(packet->payload(), + packet->payload_length_bytes()); + } + + void SetUpTest(absl::string_view codec_name, + int codec_sample_rate_hz, + int channels, + int payload_type, + int codec_frame_size_samples, + int codec_frame_size_rtp_timestamps) { + ASSERT_TRUE(SetUpSender( + channels == 1 ? kTestFileMono32kHz : kTestFileFakeStereo32kHz, 32000)); + ASSERT_TRUE(RegisterSendCodec(codec_name, codec_sample_rate_hz, channels, + payload_type, codec_frame_size_samples, + codec_frame_size_rtp_timestamps)); + } + + void SetUpTestExternalEncoder( + std::unique_ptr external_speech_encoder, + int payload_type) { + ASSERT_TRUE(send_test_); + RegisterExternalSendCodec(std::move(external_speech_encoder), payload_type); + } + + std::unique_ptr send_test_; + std::unique_ptr audio_source_; + uint32_t frame_size_rtp_timestamps_; + int packet_count_; + uint8_t payload_type_; + uint16_t last_sequence_number_; + uint32_t last_timestamp_; + std::unique_ptr payload_checksum_; + const std::string kTestFileMono32kHz = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + const std::string kTestFileFakeStereo32kHz = + webrtc::test::ResourcePath("audio_coding/testfile_fake_stereo_32kHz", + "pcm"); + const std::string kTestFileQuad48kHz = webrtc::test::ResourcePath( + "audio_coding/speech_4_channels_48k_one_second", + "wav"); +}; + +class AcmSenderBitExactnessNewApi : public AcmSenderBitExactnessOldApi {}; + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_8000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 1, 107, 80, 80)); + Run(/*audio_checksum_ref=*/"69118ed438ac76252d023e0463819471", + /*payload_checksum_ref=*/"c1edd36339ce0326cc4550041ad719a0", + /*expected_packets=*/100, + /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_16000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 16000, 1, 108, 160, 160)); + Run(/*audio_checksum_ref=*/"f95c87bdd33f631bcf80f4b19445bbd2", + /*payload_checksum_ref=*/"ad786526383178b08d80d6eee06e9bad", + /*expected_packets=*/100, + /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_32000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 32000, 1, 109, 320, 320)); + Run(/*audio_checksum_ref=*/"c50244419c5c3a2f04cc69a022c266a2", + /*payload_checksum_ref=*/"5ef82ea885e922263606c6fdbc49f651", + /*expected_packets=*/100, + /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_stereo_8000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 2, 111, 80, 80)); + Run(/*audio_checksum_ref=*/"4fccf4cc96f1e8e8de4b9fadf62ded9e", + /*payload_checksum_ref=*/"62ce5adb0d4965d0a52ec98ae7f98974", + /*expected_packets=*/100, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_stereo_16000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 16000, 2, 112, 160, 160)); + Run(/*audio_checksum_ref=*/"e15e388d9d4af8c02a59fe1552fedee3", + /*payload_checksum_ref=*/"41ca8edac4b8c71cd54fd9f25ec14870", + /*expected_packets=*/100, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcm16_stereo_32000khz_10ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 32000, 2, 113, 320, 320)); + Run(/*audio_checksum_ref=*/"b240520c0d05003fde7a174ae5957286", + /*payload_checksum_ref=*/"50e58502fb04421bf5b857dda4c96879", + /*expected_packets=*/100, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcmu_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("PCMU", 8000, 1, 0, 160, 160)); + Run(/*audio_checksum_ref=*/"c8d1fc677f33c2022ec5f83c7f302280", + /*payload_checksum_ref=*/"8f9b8750bd80fe26b6cbf6659b89f0f9", + /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcma_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("PCMA", 8000, 1, 8, 160, 160)); + Run(/*audio_checksum_ref=*/"47eb60e855eb12d1b0e6da9c975754a4", + /*payload_checksum_ref=*/"6ad745e55aa48981bfc790d0eeef2dd1", + /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcmu_stereo_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("PCMU", 8000, 2, 110, 160, 160)); + Run(/*audio_checksum_ref=*/"6ef2f57d4934714787fd0a834e3ea18e", + /*payload_checksum_ref=*/"60b6f25e8d1e74cb679cfe756dd9bca5", + /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} + +TEST_F(AcmSenderBitExactnessOldApi, Pcma_stereo_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("PCMA", 8000, 2, 118, 160, 160)); + Run(/*audio_checksum_ref=*/"a84d75e098d87ab6b260687eb4b612a2", + /*payload_checksum_ref=*/"92b282c83efd20e7eeef52ba40842cf7", + /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} + +#if defined(WEBRTC_CODEC_ILBC) && defined(WEBRTC_LINUX) && \ + defined(WEBRTC_ARCH_X86_64) +TEST_F(AcmSenderBitExactnessOldApi, Ilbc_30ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("ILBC", 8000, 1, 102, 240, 240)); + Run(/*audio_checksum_ref=*/"b14dba0de36efa5ec88a32c0b320b70f", + /*payload_checksum_ref=*/"cfae2e9f6aba96e145f2bcdd5050ce78", + /*expected_packets=*/33, + /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput); +} +#endif + +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) +TEST_F(AcmSenderBitExactnessOldApi, G722_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("G722", 16000, 1, 9, 320, 160)); + Run(/*audio_checksum_ref=*/"f5264affff25cf2cbd2e1e8a5217f9a3", + /*payload_checksum_ref=*/"fc68a87e1380614e658087cb35d5ca10", + /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput); +} +#endif + +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) +TEST_F(AcmSenderBitExactnessOldApi, G722_stereo_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("G722", 16000, 2, 119, 320, 160)); + Run(/*audio_checksum_ref=*/"be0b8528ff9db3a2219f55ddd36faf7f", + /*payload_checksum_ref=*/"66516152eeaa1e650ad94ff85f668dac", + /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} +#endif + +namespace { +// Checksum depends on libopus being compiled with or without SSE. +const std::string audio_checksum = + "6a76fe2ffba057c06eb63239b3c47abe" + "|0c4f9d33b4a7379a34ee0c0d5718afe6"; +const std::string payload_checksum = + "b43bdf7638b2bc2a5a6f30bdc640b9ed" + "|c30d463e7ed10bdd1da9045f80561f27"; +} // namespace + +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) +TEST_F(AcmSenderBitExactnessOldApi, Opus_stereo_20ms) { + ASSERT_NO_FATAL_FAILURE(SetUpTest("opus", 48000, 2, 120, 960, 960)); + Run(audio_checksum, payload_checksum, /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} +#endif + +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) +TEST_F(AcmSenderBitExactnessNewApi, OpusFromFormat_stereo_20ms) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}})); + ASSERT_TRUE(SetUpSender(kTestFileFakeStereo32kHz, 32000)); + ASSERT_NO_FATAL_FAILURE(SetUpTestExternalEncoder( + AudioEncoderOpus::MakeAudioEncoder(*config, 120), 120)); + Run(audio_checksum, payload_checksum, /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} +#endif + +// TODO(webrtc:8649): Disabled until the Encoder counterpart of +// https://webrtc-review.googlesource.com/c/src/+/129768 lands. +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) +TEST_F(AcmSenderBitExactnessNewApi, DISABLED_OpusManyChannels) { + constexpr int kNumChannels = 4; + constexpr int kOpusPayloadType = 120; + + // Read a 4 channel file at 48kHz. + ASSERT_TRUE(SetUpSender(kTestFileQuad48kHz, 48000)); + + const auto sdp_format = SdpAudioFormat("multiopus", 48000, kNumChannels, + {{"channel_mapping", "0,1,2,3"}, + {"coupled_streams", "2"}, + {"num_streams", "2"}}); + const auto encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + + ASSERT_TRUE(encoder_config.has_value()); + + ASSERT_NO_FATAL_FAILURE( + SetUpTestExternalEncoder(AudioEncoderMultiChannelOpus::MakeAudioEncoder( + *encoder_config, kOpusPayloadType), + kOpusPayloadType)); + + const auto decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format); + const auto opus_decoder = + AudioDecoderMultiChannelOpus::MakeAudioDecoder(*decoder_config); + + rtc::scoped_refptr decoder_factory = + rtc::make_ref_counted(opus_decoder.get()); + + // Set up an EXTERNAL DECODER to parse 4 channels. + Run("audio checksum check downstream|8051617907766bec5f4e4a4f7c6d5291", + "payload checksum check downstream|b09c52e44b2bdd9a0809e3a5b1623a76", + /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kQuadOutput, + decoder_factory); +} +#endif + +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) +TEST_F(AcmSenderBitExactnessNewApi, OpusFromFormat_stereo_20ms_voip) { + auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}})); + // If not set, default will be kAudio in case of stereo. + config->application = AudioEncoderOpusConfig::ApplicationMode::kVoip; + ASSERT_TRUE(SetUpSender(kTestFileFakeStereo32kHz, 32000)); + ASSERT_NO_FATAL_FAILURE(SetUpTestExternalEncoder( + AudioEncoderOpus::MakeAudioEncoder(*config, 120), 120)); + const std::string audio_maybe_sse = + "1010e60ad34cee73c939edaf563d0593" + "|c05b4523d4c3fad2bab96d2a56baa2d0"; + + const std::string payload_maybe_sse = + "ea48d94e43217793af9b7e15ece94e54" + "|bd93c492087093daf662cdd968f6cdda"; + + Run(audio_maybe_sse, payload_maybe_sse, /*expected_packets=*/50, + /*expected_channels=*/test::AcmReceiveTestOldApi::kStereoOutput); +} +#endif + +// This test is for verifying the SetBitRate function. The bitrate is changed at +// the beginning, and the number of generated bytes are checked. +class AcmSetBitRateTest : public ::testing::Test { + protected: + static const int kTestDurationMs = 1000; + + // Sets up the test::AcmSendTest object. Returns true on success, otherwise + // false. + bool SetUpSender() { + const std::string input_file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + // Note that `audio_source_` will loop forever. The test duration is set + // explicitly by `kTestDurationMs`. + audio_source_.reset(new test::InputAudioFile(input_file_name)); + static const int kSourceRateHz = 32000; + send_test_.reset(new test::AcmSendTestOldApi( + audio_source_.get(), kSourceRateHz, kTestDurationMs)); + return send_test_.get(); + } + + // Registers a send codec in the test::AcmSendTest object. Returns true on + // success, false on failure. + virtual bool RegisterSendCodec(absl::string_view payload_name, + int sampling_freq_hz, + int channels, + int payload_type, + int frame_size_samples, + int frame_size_rtp_timestamps) { + return send_test_->RegisterCodec(payload_name, sampling_freq_hz, channels, + payload_type, frame_size_samples); + } + + void RegisterExternalSendCodec( + std::unique_ptr external_speech_encoder, + int payload_type) { + send_test_->RegisterExternalCodec(std::move(external_speech_encoder)); + } + + void RunInner(int min_expected_total_bits, int max_expected_total_bits) { + int nr_bytes = 0; + while (std::unique_ptr next_packet = + send_test_->NextPacket()) { + nr_bytes += rtc::checked_cast(next_packet->payload_length_bytes()); + } + EXPECT_LE(min_expected_total_bits, nr_bytes * 8); + EXPECT_GE(max_expected_total_bits, nr_bytes * 8); + } + + void SetUpTest(absl::string_view codec_name, + int codec_sample_rate_hz, + int channels, + int payload_type, + int codec_frame_size_samples, + int codec_frame_size_rtp_timestamps) { + ASSERT_TRUE(SetUpSender()); + ASSERT_TRUE(RegisterSendCodec(codec_name, codec_sample_rate_hz, channels, + payload_type, codec_frame_size_samples, + codec_frame_size_rtp_timestamps)); + } + + std::unique_ptr send_test_; + std::unique_ptr audio_source_; +}; + +class AcmSetBitRateNewApi : public AcmSetBitRateTest { + protected: + // Runs the test. SetUpSender() must have been called and a codec must be set + // up before calling this method. + void Run(int min_expected_total_bits, int max_expected_total_bits) { + RunInner(min_expected_total_bits, max_expected_total_bits); + } +}; + +TEST_F(AcmSetBitRateNewApi, OpusFromFormat_48khz_20ms_10kbps) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"maxaveragebitrate", "10000"}})); + ASSERT_TRUE(SetUpSender()); + RegisterExternalSendCodec(AudioEncoderOpus::MakeAudioEncoder(*config, 107), + 107); + RunInner(7000, 12000); +} + +TEST_F(AcmSetBitRateNewApi, OpusFromFormat_48khz_20ms_50kbps) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"maxaveragebitrate", "50000"}})); + ASSERT_TRUE(SetUpSender()); + RegisterExternalSendCodec(AudioEncoderOpus::MakeAudioEncoder(*config, 107), + 107); + RunInner(40000, 60000); +} + +// Verify that it works when the data to send is mono and the encoder is set to +// send surround audio. +TEST_F(AudioCodingModuleTestOldApi, SendingMultiChannelForMonoInput) { + constexpr int kSampleRateHz = 48000; + constexpr int kSamplesPerChannel = kSampleRateHz * 10 / 1000; + + audio_format_ = SdpAudioFormat({"multiopus", + kSampleRateHz, + 6, + {{"minptime", "10"}, + {"useinbandfec", "1"}, + {"channel_mapping", "0,4,1,2,3,5"}, + {"num_streams", "4"}, + {"coupled_streams", "2"}}}); + + RegisterCodec(); + + input_frame_.sample_rate_hz_ = kSampleRateHz; + input_frame_.num_channels_ = 1; + input_frame_.samples_per_channel_ = kSamplesPerChannel; + for (size_t k = 0; k < 10; ++k) { + ASSERT_GE(acm_->Add10MsData(input_frame_), 0); + input_frame_.timestamp_ += kSamplesPerChannel; + } +} + +// Verify that it works when the data to send is stereo and the encoder is set +// to send surround audio. +TEST_F(AudioCodingModuleTestOldApi, SendingMultiChannelForStereoInput) { + constexpr int kSampleRateHz = 48000; + constexpr int kSamplesPerChannel = (kSampleRateHz * 10) / 1000; + + audio_format_ = SdpAudioFormat({"multiopus", + kSampleRateHz, + 6, + {{"minptime", "10"}, + {"useinbandfec", "1"}, + {"channel_mapping", "0,4,1,2,3,5"}, + {"num_streams", "4"}, + {"coupled_streams", "2"}}}); + + RegisterCodec(); + + input_frame_.sample_rate_hz_ = kSampleRateHz; + input_frame_.num_channels_ = 2; + input_frame_.samples_per_channel_ = kSamplesPerChannel; + for (size_t k = 0; k < 10; ++k) { + ASSERT_GE(acm_->Add10MsData(input_frame_), 0); + input_frame_.timestamp_ += kSamplesPerChannel; + } +} + +// Verify that it works when the data to send is mono and the encoder is set to +// send stereo audio. +TEST_F(AudioCodingModuleTestOldApi, SendingStereoForMonoInput) { + constexpr int kSampleRateHz = 48000; + constexpr int kSamplesPerChannel = (kSampleRateHz * 10) / 1000; + + audio_format_ = SdpAudioFormat("L16", kSampleRateHz, 2); + + RegisterCodec(); + + input_frame_.sample_rate_hz_ = kSampleRateHz; + input_frame_.num_channels_ = 1; + input_frame_.samples_per_channel_ = kSamplesPerChannel; + for (size_t k = 0; k < 10; ++k) { + ASSERT_GE(acm_->Add10MsData(input_frame_), 0); + input_frame_.timestamp_ += kSamplesPerChannel; + } +} + +// Verify that it works when the data to send is stereo and the encoder is set +// to send mono audio. +TEST_F(AudioCodingModuleTestOldApi, SendingMonoForStereoInput) { + constexpr int kSampleRateHz = 48000; + constexpr int kSamplesPerChannel = (kSampleRateHz * 10) / 1000; + + audio_format_ = SdpAudioFormat("L16", kSampleRateHz, 1); + + RegisterCodec(); + + input_frame_.sample_rate_hz_ = kSampleRateHz; + input_frame_.num_channels_ = 1; + input_frame_.samples_per_channel_ = kSamplesPerChannel; + for (size_t k = 0; k < 10; ++k) { + ASSERT_GE(acm_->Add10MsData(input_frame_), 0); + input_frame_.timestamp_ += kSamplesPerChannel; + } +} + +// The result on the Android platforms is inconsistent for this test case. +// On android_rel the result is different from android and android arm64 rel. +#if defined(WEBRTC_ANDROID) +#define MAYBE_OpusFromFormat_48khz_20ms_100kbps \ + DISABLED_OpusFromFormat_48khz_20ms_100kbps +#else +#define MAYBE_OpusFromFormat_48khz_20ms_100kbps \ + OpusFromFormat_48khz_20ms_100kbps +#endif +TEST_F(AcmSetBitRateNewApi, MAYBE_OpusFromFormat_48khz_20ms_100kbps) { + const auto config = AudioEncoderOpus::SdpToConfig( + SdpAudioFormat("opus", 48000, 2, {{"maxaveragebitrate", "100000"}})); + ASSERT_TRUE(SetUpSender()); + RegisterExternalSendCodec(AudioEncoderOpus::MakeAudioEncoder(*config, 107), + 107); + RunInner(80000, 120000); +} + +TEST_F(AcmSenderBitExactnessOldApi, External_Pcmu_20ms) { + AudioEncoderPcmU::Config config; + config.frame_size_ms = 20; + config.num_channels = 1; + config.payload_type = 0; + AudioEncoderPcmU encoder(config); + auto mock_encoder = std::make_unique(); + // Set expectations on the mock encoder and also delegate the calls to the + // real encoder. + EXPECT_CALL(*mock_encoder, SampleRateHz()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::SampleRateHz)); + EXPECT_CALL(*mock_encoder, NumChannels()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::NumChannels)); + EXPECT_CALL(*mock_encoder, RtpTimestampRateHz()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::RtpTimestampRateHz)); + EXPECT_CALL(*mock_encoder, Num10MsFramesInNextPacket()) + .Times(AtLeast(1)) + .WillRepeatedly( + Invoke(&encoder, &AudioEncoderPcmU::Num10MsFramesInNextPacket)); + EXPECT_CALL(*mock_encoder, GetTargetBitrate()) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke(&encoder, &AudioEncoderPcmU::GetTargetBitrate)); + EXPECT_CALL(*mock_encoder, EncodeImpl(_, _, _)) + .Times(AtLeast(1)) + .WillRepeatedly(Invoke( + &encoder, static_cast, rtc::Buffer*)>( + &AudioEncoderPcmU::Encode))); + ASSERT_TRUE(SetUpSender(kTestFileMono32kHz, 32000)); + ASSERT_NO_FATAL_FAILURE( + SetUpTestExternalEncoder(std::move(mock_encoder), config.payload_type)); + Run("c8d1fc677f33c2022ec5f83c7f302280", "8f9b8750bd80fe26b6cbf6659b89f0f9", + 50, test::AcmReceiveTestOldApi::kMonoOutput); +} + +// This test fixture is implemented to run ACM and change the desired output +// frequency during the call. The input packets are simply PCM16b-wb encoded +// payloads with a constant value of `kSampleValue`. The test fixture itself +// acts as PacketSource in between the receive test class and the constant- +// payload packet source class. The output is both written to file, and analyzed +// in this test fixture. +class AcmSwitchingOutputFrequencyOldApi : public ::testing::Test, + public test::PacketSource, + public test::AudioSink { + protected: + static const size_t kTestNumPackets = 50; + static const int kEncodedSampleRateHz = 16000; + static const size_t kPayloadLenSamples = 30 * kEncodedSampleRateHz / 1000; + static const int kPayloadType = 108; // Default payload type for PCM16b-wb. + + AcmSwitchingOutputFrequencyOldApi() + : first_output_(true), + num_packets_(0), + packet_source_(kPayloadLenSamples, + kSampleValue, + kEncodedSampleRateHz, + kPayloadType), + output_freq_2_(0), + has_toggled_(false) {} + + void Run(int output_freq_1, int output_freq_2, int toggle_period_ms) { + // Set up the receiver used to decode the packets and verify the decoded + // output. + const std::string output_file_name = + webrtc::test::OutputPath() + + ::testing::UnitTest::GetInstance() + ->current_test_info() + ->test_case_name() + + "_" + ::testing::UnitTest::GetInstance()->current_test_info()->name() + + "_output.pcm"; + test::OutputAudioFile output_file(output_file_name); + // Have the output audio sent both to file and to the WriteArray method in + // this class. + test::AudioSinkFork output(this, &output_file); + test::AcmReceiveTestToggleOutputFreqOldApi receive_test( + this, &output, output_freq_1, output_freq_2, toggle_period_ms, + test::AcmReceiveTestOldApi::kMonoOutput); + ASSERT_NO_FATAL_FAILURE(receive_test.RegisterDefaultCodecs()); + output_freq_2_ = output_freq_2; + + // This is where the actual test is executed. + receive_test.Run(); + + // Delete output file. + remove(output_file_name.c_str()); + } + + // Inherited from test::PacketSource. + std::unique_ptr NextPacket() override { + // Check if it is time to terminate the test. The packet source is of type + // ConstantPcmPacketSource, which is infinite, so we must end the test + // "manually". + if (num_packets_++ > kTestNumPackets) { + EXPECT_TRUE(has_toggled_); + return NULL; // Test ended. + } + + // Get the next packet from the source. + return packet_source_.NextPacket(); + } + + // Inherited from test::AudioSink. + bool WriteArray(const int16_t* audio, size_t num_samples) override { + // Skip checking the first output frame, since it has a number of zeros + // due to how NetEq is initialized. + if (first_output_) { + first_output_ = false; + return true; + } + for (size_t i = 0; i < num_samples; ++i) { + EXPECT_EQ(kSampleValue, audio[i]); + } + if (num_samples == + static_cast(output_freq_2_ / 100)) // Size of 10 ms frame. + has_toggled_ = true; + // The return value does not say if the values match the expectation, just + // that the method could process the samples. + return true; + } + + const int16_t kSampleValue = 1000; + bool first_output_; + size_t num_packets_; + test::ConstantPcmPacketSource packet_source_; + int output_freq_2_; + bool has_toggled_; +}; + +TEST_F(AcmSwitchingOutputFrequencyOldApi, TestWithoutToggling) { + Run(16000, 16000, 1000); +} + +TEST_F(AcmSwitchingOutputFrequencyOldApi, Toggle16KhzTo32Khz) { + Run(16000, 32000, 1000); +} + +TEST_F(AcmSwitchingOutputFrequencyOldApi, Toggle32KhzTo16Khz) { + Run(32000, 16000, 1000); +} + +TEST_F(AcmSwitchingOutputFrequencyOldApi, Toggle16KhzTo8Khz) { + Run(16000, 8000, 1000); +} + +TEST_F(AcmSwitchingOutputFrequencyOldApi, Toggle8KhzTo16Khz) { + Run(8000, 16000, 1000); +} + +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.cc b/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.cc new file mode 100644 index 0000000000..9f3bdadc88 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/call_statistics.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace acm2 { + +void CallStatistics::DecodedByNetEq(AudioFrame::SpeechType speech_type, + bool muted) { + ++decoding_stat_.calls_to_neteq; + if (muted) { + ++decoding_stat_.decoded_muted_output; + } + switch (speech_type) { + case AudioFrame::kNormalSpeech: { + ++decoding_stat_.decoded_normal; + break; + } + case AudioFrame::kPLC: { + ++decoding_stat_.decoded_neteq_plc; + break; + } + case AudioFrame::kCodecPLC: { + ++decoding_stat_.decoded_codec_plc; + break; + } + case AudioFrame::kCNG: { + ++decoding_stat_.decoded_cng; + break; + } + case AudioFrame::kPLCCNG: { + ++decoding_stat_.decoded_plc_cng; + break; + } + case AudioFrame::kUndefined: { + // If the audio is decoded by NetEq, `kUndefined` is not an option. + RTC_DCHECK_NOTREACHED(); + } + } +} + +void CallStatistics::DecodedBySilenceGenerator() { + ++decoding_stat_.calls_to_silence_generator; +} + +const AudioDecodingCallStats& CallStatistics::GetDecodingStatistics() const { + return decoding_stat_; +} + +} // namespace acm2 + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.h b/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.h new file mode 100644 index 0000000000..a2db2a29f4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_ACM2_CALL_STATISTICS_H_ +#define MODULES_AUDIO_CODING_ACM2_CALL_STATISTICS_H_ + +#include "api/audio/audio_frame.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" + +// +// This class is for book keeping of calls to ACM. It is not useful to log API +// calls which are supposed to be called every 10ms, e.g. PlayoutData10Ms(), +// however, it is useful to know the number of such calls in a given time +// interval. The current implementation covers calls to PlayoutData10Ms() with +// detailed accounting of the decoded speech type. +// +// Thread Safety +// ============= +// Please note that this class in not thread safe. The class must be protected +// if different APIs are called from different threads. +// + +namespace webrtc { + +namespace acm2 { + +class CallStatistics { + public: + CallStatistics() {} + ~CallStatistics() {} + + // Call this method to indicate that NetEq engaged in decoding. `speech_type` + // is the audio-type according to NetEq, and `muted` indicates if the decoded + // frame was produced in muted state. + void DecodedByNetEq(AudioFrame::SpeechType speech_type, bool muted); + + // Call this method to indicate that a decoding call resulted in generating + // silence, i.e. call to NetEq is bypassed and the output audio is zero. + void DecodedBySilenceGenerator(); + + // Get statistics for decoding. The statistics include the number of calls to + // NetEq and silence generator, as well as the type of speech pulled of off + // NetEq, c.f. declaration of AudioDecodingCallStats for detailed description. + const AudioDecodingCallStats& GetDecodingStatistics() const; + + private: + // Reset the decoding statistics. + void ResetDecodingStatistics(); + + AudioDecodingCallStats decoding_stat_; +}; + +} // namespace acm2 + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_ACM2_CALL_STATISTICS_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics_unittest.cc b/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics_unittest.cc new file mode 100644 index 0000000000..b96977b8e4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics_unittest.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/acm2/call_statistics.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace acm2 { + +TEST(CallStatisticsTest, InitializedZero) { + CallStatistics call_stats; + AudioDecodingCallStats stats; + + stats = call_stats.GetDecodingStatistics(); + EXPECT_EQ(0, stats.calls_to_neteq); + EXPECT_EQ(0, stats.calls_to_silence_generator); + EXPECT_EQ(0, stats.decoded_normal); + EXPECT_EQ(0, stats.decoded_cng); + EXPECT_EQ(0, stats.decoded_neteq_plc); + EXPECT_EQ(0, stats.decoded_plc_cng); + EXPECT_EQ(0, stats.decoded_muted_output); +} + +TEST(CallStatisticsTest, AllCalls) { + CallStatistics call_stats; + AudioDecodingCallStats stats; + + call_stats.DecodedBySilenceGenerator(); + call_stats.DecodedByNetEq(AudioFrame::kNormalSpeech, false); + call_stats.DecodedByNetEq(AudioFrame::kPLC, false); + call_stats.DecodedByNetEq(AudioFrame::kCodecPLC, false); + call_stats.DecodedByNetEq(AudioFrame::kPLCCNG, true); // Let this be muted. + call_stats.DecodedByNetEq(AudioFrame::kCNG, false); + + stats = call_stats.GetDecodingStatistics(); + EXPECT_EQ(5, stats.calls_to_neteq); + EXPECT_EQ(1, stats.calls_to_silence_generator); + EXPECT_EQ(1, stats.decoded_normal); + EXPECT_EQ(1, stats.decoded_cng); + EXPECT_EQ(1, stats.decoded_neteq_plc); + EXPECT_EQ(1, stats.decoded_codec_plc); + EXPECT_EQ(1, stats.decoded_plc_cng); + EXPECT_EQ(1, stats.decoded_muted_output); +} + +} // namespace acm2 + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_coding.gni b/third_party/libwebrtc/modules/audio_coding/audio_coding.gni new file mode 100644 index 0000000000..3b147091de --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_coding.gni @@ -0,0 +1,25 @@ +# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +audio_codec_defines = [] +if (rtc_include_ilbc) { + audio_codec_defines += [ "WEBRTC_CODEC_ILBC" ] +} +if (rtc_include_opus) { + audio_codec_defines += [ "WEBRTC_CODEC_OPUS" ] +} +if (rtc_opus_support_120ms_ptime) { + audio_codec_defines += [ "WEBRTC_OPUS_SUPPORT_120MS_PTIME=1" ] +} else { + audio_codec_defines += [ "WEBRTC_OPUS_SUPPORT_120MS_PTIME=0" ] +} + +audio_coding_defines = audio_codec_defines +neteq_defines = audio_codec_defines diff --git a/third_party/libwebrtc/modules/audio_coding/audio_coding_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/audio_coding_gn/moz.build new file mode 100644 index 0000000000..03d99c93b8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_coding_gn/moz.build @@ -0,0 +1,236 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/acm2/acm_receiver.cc", + "/third_party/libwebrtc/modules/audio_coding/acm2/acm_remixing.cc", + "/third_party/libwebrtc/modules/audio_coding/acm2/acm_resampler.cc", + "/third_party/libwebrtc/modules/audio_coding/acm2/audio_coding_module.cc", + "/third_party/libwebrtc/modules/audio_coding/acm2/call_statistics.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("audio_coding_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/audio_coding_module_typedefs_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/audio_coding_module_typedefs_gn/moz.build new file mode 100644 index 0000000000..2826ed8642 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_coding_module_typedefs_gn/moz.build @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("audio_coding_module_typedefs_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/audio_coding_opus_common_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/audio_coding_opus_common_gn/moz.build new file mode 100644 index 0000000000..bf852e8442 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_coding_opus_common_gn/moz.build @@ -0,0 +1,225 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("audio_coding_opus_common_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/audio_encoder_cng_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/audio_encoder_cng_gn/moz.build new file mode 100644 index 0000000000..56b6d50f6c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_encoder_cng_gn/moz.build @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("audio_encoder_cng_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc new file mode 100644 index 0000000000..16fd2a1b9a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { + +AudioEncoderRuntimeConfig::AudioEncoderRuntimeConfig() = default; + +AudioEncoderRuntimeConfig::AudioEncoderRuntimeConfig( + const AudioEncoderRuntimeConfig& other) = default; + +AudioEncoderRuntimeConfig::~AudioEncoderRuntimeConfig() = default; + +AudioEncoderRuntimeConfig& AudioEncoderRuntimeConfig::operator=( + const AudioEncoderRuntimeConfig& other) = default; + +bool AudioEncoderRuntimeConfig::operator==( + const AudioEncoderRuntimeConfig& other) const { + return bitrate_bps == other.bitrate_bps && + frame_length_ms == other.frame_length_ms && + uplink_packet_loss_fraction == other.uplink_packet_loss_fraction && + enable_fec == other.enable_fec && enable_dtx == other.enable_dtx && + num_channels == other.num_channels; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc new file mode 100644 index 0000000000..64163f9118 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" + +#include + +#include +#include + +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" +#include "modules/audio_coding/audio_network_adaptor/event_log_writer.h" +#include "rtc_base/checks.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { +constexpr int kEventLogMinBitrateChangeBps = 5000; +constexpr float kEventLogMinBitrateChangeFraction = 0.25; +constexpr float kEventLogMinPacketLossChangeFraction = 0.5; +} // namespace + +AudioNetworkAdaptorImpl::Config::Config() : event_log(nullptr) {} + +AudioNetworkAdaptorImpl::Config::~Config() = default; + +AudioNetworkAdaptorImpl::AudioNetworkAdaptorImpl( + const Config& config, + std::unique_ptr controller_manager, + std::unique_ptr debug_dump_writer) + : config_(config), + controller_manager_(std::move(controller_manager)), + debug_dump_writer_(std::move(debug_dump_writer)), + event_log_writer_( + config.event_log + ? new EventLogWriter(config.event_log, + kEventLogMinBitrateChangeBps, + kEventLogMinBitrateChangeFraction, + kEventLogMinPacketLossChangeFraction) + : nullptr) { + RTC_DCHECK(controller_manager_); +} + +AudioNetworkAdaptorImpl::~AudioNetworkAdaptorImpl() = default; + +void AudioNetworkAdaptorImpl::SetUplinkBandwidth(int uplink_bandwidth_bps) { + last_metrics_.uplink_bandwidth_bps = uplink_bandwidth_bps; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + last_metrics_.uplink_packet_loss_fraction = uplink_packet_loss_fraction; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_packet_loss_fraction = uplink_packet_loss_fraction; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetRtt(int rtt_ms) { + last_metrics_.rtt_ms = rtt_ms; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.rtt_ms = rtt_ms; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetTargetAudioBitrate( + int target_audio_bitrate_bps) { + last_metrics_.target_audio_bitrate_bps = target_audio_bitrate_bps; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.target_audio_bitrate_bps = target_audio_bitrate_bps; + UpdateNetworkMetrics(network_metrics); +} + +void AudioNetworkAdaptorImpl::SetOverhead(size_t overhead_bytes_per_packet) { + last_metrics_.overhead_bytes_per_packet = overhead_bytes_per_packet; + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + UpdateNetworkMetrics(network_metrics); +} + +AudioEncoderRuntimeConfig AudioNetworkAdaptorImpl::GetEncoderRuntimeConfig() { + AudioEncoderRuntimeConfig config; + for (auto& controller : + controller_manager_->GetSortedControllers(last_metrics_)) + controller->MakeDecision(&config); + + // Update ANA stats. + auto increment_opt = [](absl::optional& a) { + a = a.value_or(0) + 1; + }; + if (prev_config_) { + if (config.bitrate_bps != prev_config_->bitrate_bps) { + increment_opt(stats_.bitrate_action_counter); + } + if (config.enable_dtx != prev_config_->enable_dtx) { + increment_opt(stats_.dtx_action_counter); + } + if (config.enable_fec != prev_config_->enable_fec) { + increment_opt(stats_.fec_action_counter); + } + if (config.frame_length_ms && prev_config_->frame_length_ms) { + if (*config.frame_length_ms > *prev_config_->frame_length_ms) { + increment_opt(stats_.frame_length_increase_counter); + } else if (*config.frame_length_ms < *prev_config_->frame_length_ms) { + increment_opt(stats_.frame_length_decrease_counter); + } + } + if (config.num_channels != prev_config_->num_channels) { + increment_opt(stats_.channel_action_counter); + } + if (config.uplink_packet_loss_fraction) { + stats_.uplink_packet_loss_fraction = *config.uplink_packet_loss_fraction; + } + } + prev_config_ = config; + + if (debug_dump_writer_) + debug_dump_writer_->DumpEncoderRuntimeConfig(config, rtc::TimeMillis()); + + if (event_log_writer_) + event_log_writer_->MaybeLogEncoderConfig(config); + + return config; +} + +void AudioNetworkAdaptorImpl::StartDebugDump(FILE* file_handle) { + debug_dump_writer_ = DebugDumpWriter::Create(file_handle); +} + +void AudioNetworkAdaptorImpl::StopDebugDump() { + debug_dump_writer_.reset(nullptr); +} + +ANAStats AudioNetworkAdaptorImpl::GetStats() const { + return stats_; +} + +void AudioNetworkAdaptorImpl::DumpNetworkMetrics() { + if (debug_dump_writer_) + debug_dump_writer_->DumpNetworkMetrics(last_metrics_, rtc::TimeMillis()); +} + +void AudioNetworkAdaptorImpl::UpdateNetworkMetrics( + const Controller::NetworkMetrics& network_metrics) { + for (auto& controller : controller_manager_->GetControllers()) + controller->UpdateNetworkMetrics(network_metrics); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h new file mode 100644 index 0000000000..664e76bda5 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_AUDIO_NETWORK_ADAPTOR_IMPL_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_AUDIO_NETWORK_ADAPTOR_IMPL_H_ + +#include + +#include + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_encoder.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { + +class ControllerManager; +class EventLogWriter; +class RtcEventLog; + +class AudioNetworkAdaptorImpl final : public AudioNetworkAdaptor { + public: + struct Config { + Config(); + ~Config(); + RtcEventLog* event_log; + }; + + AudioNetworkAdaptorImpl( + const Config& config, + std::unique_ptr controller_manager, + std::unique_ptr debug_dump_writer = nullptr); + + ~AudioNetworkAdaptorImpl() override; + + AudioNetworkAdaptorImpl(const AudioNetworkAdaptorImpl&) = delete; + AudioNetworkAdaptorImpl& operator=(const AudioNetworkAdaptorImpl&) = delete; + + void SetUplinkBandwidth(int uplink_bandwidth_bps) override; + + void SetUplinkPacketLossFraction(float uplink_packet_loss_fraction) override; + + void SetRtt(int rtt_ms) override; + + void SetTargetAudioBitrate(int target_audio_bitrate_bps) override; + + void SetOverhead(size_t overhead_bytes_per_packet) override; + + AudioEncoderRuntimeConfig GetEncoderRuntimeConfig() override; + + void StartDebugDump(FILE* file_handle) override; + + void StopDebugDump() override; + + ANAStats GetStats() const override; + + private: + void DumpNetworkMetrics(); + + void UpdateNetworkMetrics(const Controller::NetworkMetrics& network_metrics); + + const Config config_; + + std::unique_ptr controller_manager_; + + std::unique_ptr debug_dump_writer_; + + const std::unique_ptr event_log_writer_; + + Controller::NetworkMetrics last_metrics_; + + absl::optional prev_config_; + + ANAStats stats_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_AUDIO_NETWORK_ADAPTOR_IMPL_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc new file mode 100644 index 0000000000..70a50d6de7 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" + +#include +#include + +#include "api/rtc_event_log/rtc_event.h" +#include "logging/rtc_event_log/events/rtc_event_audio_network_adaptation.h" +#include "logging/rtc_event_log/mock/mock_rtc_event_log.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_controller.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h" +#include "rtc_base/fake_clock.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::SetArgPointee; + +namespace { + +constexpr size_t kNumControllers = 2; + +constexpr int64_t kClockInitialTimeMs = 12345678; + +MATCHER_P(NetworkMetricsIs, metric, "") { + return arg.uplink_bandwidth_bps == metric.uplink_bandwidth_bps && + arg.target_audio_bitrate_bps == metric.target_audio_bitrate_bps && + arg.rtt_ms == metric.rtt_ms && + arg.overhead_bytes_per_packet == metric.overhead_bytes_per_packet && + arg.uplink_packet_loss_fraction == metric.uplink_packet_loss_fraction; +} + +MATCHER_P(IsRtcEventAnaConfigEqualTo, config, "") { + if (arg->GetType() != RtcEvent::Type::AudioNetworkAdaptation) { + return false; + } + auto ana_event = static_cast(arg); + return ana_event->config() == config; +} + +MATCHER_P(EncoderRuntimeConfigIs, config, "") { + return arg.bitrate_bps == config.bitrate_bps && + arg.frame_length_ms == config.frame_length_ms && + arg.uplink_packet_loss_fraction == + config.uplink_packet_loss_fraction && + arg.enable_fec == config.enable_fec && + arg.enable_dtx == config.enable_dtx && + arg.num_channels == config.num_channels; +} + +struct AudioNetworkAdaptorStates { + std::unique_ptr audio_network_adaptor; + std::vector> mock_controllers; + std::unique_ptr event_log; + MockDebugDumpWriter* mock_debug_dump_writer; +}; + +AudioNetworkAdaptorStates CreateAudioNetworkAdaptor() { + AudioNetworkAdaptorStates states; + std::vector controllers; + for (size_t i = 0; i < kNumControllers; ++i) { + auto controller = + std::unique_ptr(new NiceMock()); + EXPECT_CALL(*controller, Die()); + controllers.push_back(controller.get()); + states.mock_controllers.push_back(std::move(controller)); + } + + auto controller_manager = std::unique_ptr( + new NiceMock()); + + EXPECT_CALL(*controller_manager, Die()); + EXPECT_CALL(*controller_manager, GetControllers()) + .WillRepeatedly(Return(controllers)); + EXPECT_CALL(*controller_manager, GetSortedControllers(_)) + .WillRepeatedly(Return(controllers)); + + states.event_log.reset(new NiceMock()); + + auto debug_dump_writer = + std::unique_ptr(new NiceMock()); + EXPECT_CALL(*debug_dump_writer, Die()); + states.mock_debug_dump_writer = debug_dump_writer.get(); + + AudioNetworkAdaptorImpl::Config config; + config.event_log = states.event_log.get(); + // AudioNetworkAdaptorImpl governs the lifetime of controller manager. + states.audio_network_adaptor.reset(new AudioNetworkAdaptorImpl( + config, std::move(controller_manager), std::move(debug_dump_writer))); + + return states; +} + +void SetExpectCallToUpdateNetworkMetrics( + const std::vector>& controllers, + const Controller::NetworkMetrics& check) { + for (auto& mock_controller : controllers) { + EXPECT_CALL(*mock_controller, + UpdateNetworkMetrics(NetworkMetricsIs(check))); + } +} + +} // namespace + +TEST(AudioNetworkAdaptorImplTest, + UpdateNetworkMetricsIsCalledOnSetUplinkBandwidth) { + auto states = CreateAudioNetworkAdaptor(); + constexpr int kBandwidth = 16000; + Controller::NetworkMetrics check; + check.uplink_bandwidth_bps = kBandwidth; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetUplinkBandwidth(kBandwidth); +} + +TEST(AudioNetworkAdaptorImplTest, + UpdateNetworkMetricsIsCalledOnSetUplinkPacketLossFraction) { + auto states = CreateAudioNetworkAdaptor(); + constexpr float kPacketLoss = 0.7f; + Controller::NetworkMetrics check; + check.uplink_packet_loss_fraction = kPacketLoss; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetUplinkPacketLossFraction(kPacketLoss); +} + +TEST(AudioNetworkAdaptorImplTest, UpdateNetworkMetricsIsCalledOnSetRtt) { + auto states = CreateAudioNetworkAdaptor(); + constexpr int kRtt = 100; + Controller::NetworkMetrics check; + check.rtt_ms = kRtt; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetRtt(kRtt); +} + +TEST(AudioNetworkAdaptorImplTest, + UpdateNetworkMetricsIsCalledOnSetTargetAudioBitrate) { + auto states = CreateAudioNetworkAdaptor(); + constexpr int kTargetAudioBitrate = 15000; + Controller::NetworkMetrics check; + check.target_audio_bitrate_bps = kTargetAudioBitrate; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetTargetAudioBitrate(kTargetAudioBitrate); +} + +TEST(AudioNetworkAdaptorImplTest, UpdateNetworkMetricsIsCalledOnSetOverhead) { + auto states = CreateAudioNetworkAdaptor(); + constexpr size_t kOverhead = 64; + Controller::NetworkMetrics check; + check.overhead_bytes_per_packet = kOverhead; + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetOverhead(kOverhead); +} + +TEST(AudioNetworkAdaptorImplTest, + MakeDecisionIsCalledOnGetEncoderRuntimeConfig) { + auto states = CreateAudioNetworkAdaptor(); + for (auto& mock_controller : states.mock_controllers) + EXPECT_CALL(*mock_controller, MakeDecision(_)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); +} + +TEST(AudioNetworkAdaptorImplTest, + DumpEncoderRuntimeConfigIsCalledOnGetEncoderRuntimeConfig) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-BitrateAdaptation/Enabled/WebRTC-Audio-FecAdaptation/" + "Enabled/"); + rtc::ScopedFakeClock fake_clock; + fake_clock.AdvanceTime(TimeDelta::Millis(kClockInitialTimeMs)); + auto states = CreateAudioNetworkAdaptor(); + AudioEncoderRuntimeConfig config; + config.bitrate_bps = 32000; + config.enable_fec = true; + + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config)); + + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpEncoderRuntimeConfig(EncoderRuntimeConfigIs(config), + kClockInitialTimeMs)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); +} + +TEST(AudioNetworkAdaptorImplTest, + DumpNetworkMetricsIsCalledOnSetNetworkMetrics) { + rtc::ScopedFakeClock fake_clock; + fake_clock.AdvanceTime(TimeDelta::Millis(kClockInitialTimeMs)); + + auto states = CreateAudioNetworkAdaptor(); + + constexpr int kBandwidth = 16000; + constexpr float kPacketLoss = 0.7f; + constexpr int kRtt = 100; + constexpr int kTargetAudioBitrate = 15000; + constexpr size_t kOverhead = 64; + + Controller::NetworkMetrics check; + check.uplink_bandwidth_bps = kBandwidth; + int64_t timestamp_check = kClockInitialTimeMs; + + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetUplinkBandwidth(kBandwidth); + + fake_clock.AdvanceTime(TimeDelta::Millis(100)); + timestamp_check += 100; + check.uplink_packet_loss_fraction = kPacketLoss; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetUplinkPacketLossFraction(kPacketLoss); + + fake_clock.AdvanceTime(TimeDelta::Millis(50)); + timestamp_check += 50; + + fake_clock.AdvanceTime(TimeDelta::Millis(200)); + timestamp_check += 200; + check.rtt_ms = kRtt; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetRtt(kRtt); + + fake_clock.AdvanceTime(TimeDelta::Millis(150)); + timestamp_check += 150; + check.target_audio_bitrate_bps = kTargetAudioBitrate; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetTargetAudioBitrate(kTargetAudioBitrate); + + fake_clock.AdvanceTime(TimeDelta::Millis(50)); + timestamp_check += 50; + check.overhead_bytes_per_packet = kOverhead; + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetOverhead(kOverhead); +} + +TEST(AudioNetworkAdaptorImplTest, LogRuntimeConfigOnGetEncoderRuntimeConfig) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-BitrateAdaptation/Enabled/WebRTC-Audio-FecAdaptation/" + "Enabled/"); + auto states = CreateAudioNetworkAdaptor(); + + AudioEncoderRuntimeConfig config; + config.bitrate_bps = 32000; + config.enable_fec = true; + + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config)); + + EXPECT_CALL(*states.event_log, LogProxy(IsRtcEventAnaConfigEqualTo(config))) + .Times(1); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); +} + +TEST(AudioNetworkAdaptorImplTest, TestANAStats) { + auto states = CreateAudioNetworkAdaptor(); + + // Simulate some adaptation, otherwise the stats will not show anything. + AudioEncoderRuntimeConfig config1, config2; + config1.bitrate_bps = 32000; + config1.num_channels = 2; + config1.enable_fec = true; + config1.enable_dtx = true; + config1.frame_length_ms = 120; + config1.uplink_packet_loss_fraction = 0.1f; + config2.bitrate_bps = 16000; + config2.num_channels = 1; + config2.enable_fec = false; + config2.enable_dtx = false; + config2.frame_length_ms = 60; + config1.uplink_packet_loss_fraction = 0.1f; + + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config1)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config2)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); + EXPECT_CALL(*states.mock_controllers[0], MakeDecision(_)) + .WillOnce(SetArgPointee<0>(config1)); + states.audio_network_adaptor->GetEncoderRuntimeConfig(); + + auto ana_stats = states.audio_network_adaptor->GetStats(); + + EXPECT_EQ(ana_stats.bitrate_action_counter, 2u); + EXPECT_EQ(ana_stats.channel_action_counter, 2u); + EXPECT_EQ(ana_stats.dtx_action_counter, 2u); + EXPECT_EQ(ana_stats.fec_action_counter, 2u); + EXPECT_EQ(ana_stats.frame_length_increase_counter, 1u); + EXPECT_EQ(ana_stats.frame_length_decrease_counter, 1u); + EXPECT_EQ(ana_stats.uplink_packet_loss_fraction, 0.1f); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc new file mode 100644 index 0000000000..88ca38d074 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/bitrate_controller.h" + +#include + +#include "rtc_base/checks.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace audio_network_adaptor { + +BitrateController::Config::Config(int initial_bitrate_bps, + int initial_frame_length_ms, + int fl_increase_overhead_offset, + int fl_decrease_overhead_offset) + : initial_bitrate_bps(initial_bitrate_bps), + initial_frame_length_ms(initial_frame_length_ms), + fl_increase_overhead_offset(fl_increase_overhead_offset), + fl_decrease_overhead_offset(fl_decrease_overhead_offset) {} + +BitrateController::Config::~Config() = default; + +BitrateController::BitrateController(const Config& config) + : config_(config), + bitrate_bps_(config_.initial_bitrate_bps), + frame_length_ms_(config_.initial_frame_length_ms) { + RTC_DCHECK_GT(bitrate_bps_, 0); + RTC_DCHECK_GT(frame_length_ms_, 0); +} + +BitrateController::~BitrateController() = default; + +void BitrateController::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.target_audio_bitrate_bps) + target_audio_bitrate_bps_ = network_metrics.target_audio_bitrate_bps; + if (network_metrics.overhead_bytes_per_packet) { + RTC_DCHECK_GT(*network_metrics.overhead_bytes_per_packet, 0); + overhead_bytes_per_packet_ = network_metrics.overhead_bytes_per_packet; + } +} + +void BitrateController::MakeDecision(AudioEncoderRuntimeConfig* config) { + // Decision on `bitrate_bps` should not have been made. + RTC_DCHECK(!config->bitrate_bps); + if (target_audio_bitrate_bps_ && overhead_bytes_per_packet_) { + if (config->frame_length_ms) + frame_length_ms_ = *config->frame_length_ms; + int offset = config->last_fl_change_increase + ? config_.fl_increase_overhead_offset + : config_.fl_decrease_overhead_offset; + // Check that + // -(*overhead_bytes_per_packet_) <= offset <= (*overhead_bytes_per_packet_) + RTC_DCHECK_GE(*overhead_bytes_per_packet_, -offset); + RTC_DCHECK_LE(offset, *overhead_bytes_per_packet_); + int overhead_rate_bps = static_cast( + (*overhead_bytes_per_packet_ + offset) * 8 * 1000 / frame_length_ms_); + bitrate_bps_ = std::max(0, *target_audio_bitrate_bps_ - overhead_rate_bps); + } + config->bitrate_bps = bitrate_bps_; +} + +} // namespace audio_network_adaptor +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.h new file mode 100644 index 0000000000..c1032146cc --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_BITRATE_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_BITRATE_CONTROLLER_H_ + +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { +namespace audio_network_adaptor { + +class BitrateController final : public Controller { + public: + struct Config { + Config(int initial_bitrate_bps, + int initial_frame_length_ms, + int fl_increase_overhead_offset, + int fl_decrease_overhead_offset); + ~Config(); + int initial_bitrate_bps; + int initial_frame_length_ms; + int fl_increase_overhead_offset; + int fl_decrease_overhead_offset; + }; + + explicit BitrateController(const Config& config); + + ~BitrateController() override; + + BitrateController(const BitrateController&) = delete; + BitrateController& operator=(const BitrateController&) = delete; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + const Config config_; + int bitrate_bps_; + int frame_length_ms_; + absl::optional target_audio_bitrate_bps_; + absl::optional overhead_bytes_per_packet_; +}; + +} // namespace audio_network_adaptor +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_BITRATE_CONTROLLER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller_unittest.cc new file mode 100644 index 0000000000..9c593b818b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller_unittest.cc @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/bitrate_controller.h" + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +namespace audio_network_adaptor { + +namespace { + +void UpdateNetworkMetrics( + BitrateController* controller, + const absl::optional& target_audio_bitrate_bps, + const absl::optional& overhead_bytes_per_packet) { + // UpdateNetworkMetrics can accept multiple network metric updates at once. + // However, currently, the most used case is to update one metric at a time. + // To reflect this fact, we separate the calls. + if (target_audio_bitrate_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.target_audio_bitrate_bps = target_audio_bitrate_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + if (overhead_bytes_per_packet) { + Controller::NetworkMetrics network_metrics; + network_metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + controller->UpdateNetworkMetrics(network_metrics); + } +} + +void CheckDecision(BitrateController* controller, + const absl::optional& frame_length_ms, + int expected_bitrate_bps) { + AudioEncoderRuntimeConfig config; + config.frame_length_ms = frame_length_ms; + controller->MakeDecision(&config); + EXPECT_EQ(expected_bitrate_bps, config.bitrate_bps); +} + +} // namespace + +// These tests are named AnaBitrateControllerTest to distinguish from +// BitrateControllerTest in +// modules/bitrate_controller/bitrate_controller_unittest.cc. + +TEST(AnaBitrateControllerTest, OutputInitValueWhenTargetBitrateUnknown) { + constexpr int kInitialBitrateBps = 32000; + constexpr int kInitialFrameLengthMs = 20; + constexpr size_t kOverheadBytesPerPacket = 64; + BitrateController controller(BitrateController::Config( + kInitialBitrateBps, kInitialFrameLengthMs, 0, 0)); + UpdateNetworkMetrics(&controller, absl::nullopt, kOverheadBytesPerPacket); + CheckDecision(&controller, kInitialFrameLengthMs * 2, kInitialBitrateBps); +} + +TEST(AnaBitrateControllerTest, OutputInitValueWhenOverheadUnknown) { + constexpr int kInitialBitrateBps = 32000; + constexpr int kInitialFrameLengthMs = 20; + constexpr int kTargetBitrateBps = 48000; + BitrateController controller(BitrateController::Config( + kInitialBitrateBps, kInitialFrameLengthMs, 0, 0)); + UpdateNetworkMetrics(&controller, kTargetBitrateBps, absl::nullopt); + CheckDecision(&controller, kInitialFrameLengthMs * 2, kInitialBitrateBps); +} + +TEST(AnaBitrateControllerTest, ChangeBitrateOnTargetBitrateChanged) { + constexpr int kInitialFrameLengthMs = 20; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = kTargetBitrateBps - kOverheadBytesPerPacket * 8 * + 1000 / + kInitialFrameLengthMs; + // Frame length unchanged, bitrate changes in accordance with + // `metrics.target_audio_bitrate_bps` and `metrics.overhead_bytes_per_packet`. + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, kInitialFrameLengthMs, kBitrateBps); +} + +TEST(AnaBitrateControllerTest, UpdateMultipleNetworkMetricsAtOnce) { + // This test is similar to ChangeBitrateOnTargetBitrateChanged. But instead of + // using ::UpdateNetworkMetrics(...), which calls + // BitrateController::UpdateNetworkMetrics(...) multiple times, we + // we call it only once. This is to verify that + // BitrateController::UpdateNetworkMetrics(...) can handle multiple + // network updates at once. This is, however, not a common use case in current + // audio_network_adaptor_impl.cc. + constexpr int kInitialFrameLengthMs = 20; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = kTargetBitrateBps - kOverheadBytesPerPacket * 8 * + 1000 / + kInitialFrameLengthMs; + Controller::NetworkMetrics network_metrics; + network_metrics.target_audio_bitrate_bps = kTargetBitrateBps; + network_metrics.overhead_bytes_per_packet = kOverheadBytesPerPacket; + controller.UpdateNetworkMetrics(network_metrics); + CheckDecision(&controller, kInitialFrameLengthMs, kBitrateBps); +} + +TEST(AnaBitrateControllerTest, TreatUnknownFrameLengthAsFrameLengthUnchanged) { + constexpr int kInitialFrameLengthMs = 20; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = kTargetBitrateBps - kOverheadBytesPerPacket * 8 * + 1000 / + kInitialFrameLengthMs; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, absl::nullopt, kBitrateBps); +} + +TEST(AnaBitrateControllerTest, IncreaseBitrateOnFrameLengthIncreased) { + constexpr int kInitialFrameLengthMs = 20; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = kTargetBitrateBps - kOverheadBytesPerPacket * 8 * + 1000 / + kInitialFrameLengthMs; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, absl::nullopt, kBitrateBps); + + constexpr int kFrameLengthMs = 60; + constexpr size_t kPacketOverheadRateDiff = + kOverheadBytesPerPacket * 8 * 1000 / 20 - + kOverheadBytesPerPacket * 8 * 1000 / 60; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, kFrameLengthMs, + kBitrateBps + kPacketOverheadRateDiff); +} + +TEST(AnaBitrateControllerTest, DecreaseBitrateOnFrameLengthDecreased) { + constexpr int kInitialFrameLengthMs = 60; + BitrateController controller( + BitrateController::Config(32000, kInitialFrameLengthMs, 0, 0)); + + constexpr int kTargetBitrateBps = 48000; + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kBitrateBps = kTargetBitrateBps - kOverheadBytesPerPacket * 8 * + 1000 / + kInitialFrameLengthMs; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, absl::nullopt, kBitrateBps); + + constexpr int kFrameLengthMs = 20; + constexpr size_t kPacketOverheadRateDiff = + kOverheadBytesPerPacket * 8 * 1000 / 20 - + kOverheadBytesPerPacket * 8 * 1000 / 60; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, kFrameLengthMs, + kBitrateBps - kPacketOverheadRateDiff); +} + +TEST(AnaBitrateControllerTest, BitrateNeverBecomesNegative) { + BitrateController controller(BitrateController::Config(32000, 20, 0, 0)); + constexpr size_t kOverheadBytesPerPacket = 64; + constexpr int kFrameLengthMs = 60; + // Set a target rate smaller than overhead rate, the bitrate is bounded by 0. + constexpr int kTargetBitrateBps = + kOverheadBytesPerPacket * 8 * 1000 / kFrameLengthMs - 1; + UpdateNetworkMetrics(&controller, kTargetBitrateBps, kOverheadBytesPerPacket); + CheckDecision(&controller, kFrameLengthMs, 0); +} + +TEST(AnaBitrateControllerTest, CheckBehaviorOnChangingCondition) { + BitrateController controller(BitrateController::Config(32000, 20, 0, 0)); + + // Start from an arbitrary overall bitrate. + int overall_bitrate = 34567; + size_t overhead_bytes_per_packet = 64; + int frame_length_ms = 20; + int current_bitrate = rtc::checked_cast( + overall_bitrate - overhead_bytes_per_packet * 8 * 1000 / frame_length_ms); + + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: increase overall bitrate. + overall_bitrate += 100; + current_bitrate += 100; + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: change frame length. + frame_length_ms = 60; + current_bitrate += + rtc::checked_cast(overhead_bytes_per_packet * 8 * 1000 / 20 - + overhead_bytes_per_packet * 8 * 1000 / 60); + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: change overhead. + overhead_bytes_per_packet -= 30; + current_bitrate += 30 * 8 * 1000 / frame_length_ms; + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: change frame length. + frame_length_ms = 20; + current_bitrate -= + rtc::checked_cast(overhead_bytes_per_packet * 8 * 1000 / 20 - + overhead_bytes_per_packet * 8 * 1000 / 60); + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); + + // Next: decrease overall bitrate and frame length. + overall_bitrate -= 100; + current_bitrate -= 100; + frame_length_ms = 60; + current_bitrate += + rtc::checked_cast(overhead_bytes_per_packet * 8 * 1000 / 20 - + overhead_bytes_per_packet * 8 * 1000 / 60); + + UpdateNetworkMetrics(&controller, overall_bitrate, overhead_bytes_per_packet); + CheckDecision(&controller, frame_length_ms, current_bitrate); +} + +} // namespace audio_network_adaptor +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc new file mode 100644 index 0000000000..2ef2f4c4d8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/channel_controller.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +ChannelController::Config::Config(size_t num_encoder_channels, + size_t intial_channels_to_encode, + int channel_1_to_2_bandwidth_bps, + int channel_2_to_1_bandwidth_bps) + : num_encoder_channels(num_encoder_channels), + intial_channels_to_encode(intial_channels_to_encode), + channel_1_to_2_bandwidth_bps(channel_1_to_2_bandwidth_bps), + channel_2_to_1_bandwidth_bps(channel_2_to_1_bandwidth_bps) {} + +ChannelController::ChannelController(const Config& config) + : config_(config), channels_to_encode_(config_.intial_channels_to_encode) { + RTC_DCHECK_GT(config_.intial_channels_to_encode, 0lu); + // Currently, we require `intial_channels_to_encode` to be <= 2. + RTC_DCHECK_LE(config_.intial_channels_to_encode, 2lu); + RTC_DCHECK_GE(config_.num_encoder_channels, + config_.intial_channels_to_encode); +} + +ChannelController::~ChannelController() = default; + +void ChannelController::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; +} + +void ChannelController::MakeDecision(AudioEncoderRuntimeConfig* config) { + // Decision on `num_channels` should not have been made. + RTC_DCHECK(!config->num_channels); + + if (uplink_bandwidth_bps_) { + if (channels_to_encode_ == 2 && + *uplink_bandwidth_bps_ <= config_.channel_2_to_1_bandwidth_bps) { + channels_to_encode_ = 1; + } else if (channels_to_encode_ == 1 && + *uplink_bandwidth_bps_ >= config_.channel_1_to_2_bandwidth_bps) { + channels_to_encode_ = + std::min(static_cast(2), config_.num_encoder_channels); + } + } + config->num_channels = channels_to_encode_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.h new file mode 100644 index 0000000000..3cd4bb7dec --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CHANNEL_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CHANNEL_CONTROLLER_H_ + +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { + +class ChannelController final : public Controller { + public: + struct Config { + Config(size_t num_encoder_channels, + size_t intial_channels_to_encode, + int channel_1_to_2_bandwidth_bps, + int channel_2_to_1_bandwidth_bps); + size_t num_encoder_channels; + size_t intial_channels_to_encode; + // Uplink bandwidth above which the number of encoded channels should switch + // from 1 to 2. + int channel_1_to_2_bandwidth_bps; + // Uplink bandwidth below which the number of encoded channels should switch + // from 2 to 1. + int channel_2_to_1_bandwidth_bps; + }; + + explicit ChannelController(const Config& config); + + ~ChannelController() override; + + ChannelController(const ChannelController&) = delete; + ChannelController& operator=(const ChannelController&) = delete; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + const Config config_; + size_t channels_to_encode_; + absl::optional uplink_bandwidth_bps_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CHANNEL_CONTROLLER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller_unittest.cc new file mode 100644 index 0000000000..21504bcec0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller_unittest.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/channel_controller.h" + +#include + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kNumChannels = 2; +constexpr int kChannel1To2BandwidthBps = 31000; +constexpr int kChannel2To1BandwidthBps = 29000; +constexpr int kMediumBandwidthBps = + (kChannel1To2BandwidthBps + kChannel2To1BandwidthBps) / 2; + +std::unique_ptr CreateChannelController(int init_channels) { + std::unique_ptr controller( + new ChannelController(ChannelController::Config( + kNumChannels, init_channels, kChannel1To2BandwidthBps, + kChannel2To1BandwidthBps))); + return controller; +} + +void CheckDecision(ChannelController* controller, + const absl::optional& uplink_bandwidth_bps, + size_t expected_num_channels) { + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + AudioEncoderRuntimeConfig config; + controller->MakeDecision(&config); + EXPECT_EQ(expected_num_channels, config.num_channels); +} + +} // namespace + +TEST(ChannelControllerTest, OutputInitValueWhenUplinkBandwidthUnknown) { + constexpr int kInitChannels = 2; + auto controller = CreateChannelController(kInitChannels); + CheckDecision(controller.get(), absl::nullopt, kInitChannels); +} + +TEST(ChannelControllerTest, SwitchTo2ChannelsOnHighUplinkBandwidth) { + constexpr int kInitChannels = 1; + auto controller = CreateChannelController(kInitChannels); + // Use high bandwidth to check output switch to 2. + CheckDecision(controller.get(), kChannel1To2BandwidthBps, 2); +} + +TEST(ChannelControllerTest, SwitchTo1ChannelOnLowUplinkBandwidth) { + constexpr int kInitChannels = 2; + auto controller = CreateChannelController(kInitChannels); + // Use low bandwidth to check output switch to 1. + CheckDecision(controller.get(), kChannel2To1BandwidthBps, 1); +} + +TEST(ChannelControllerTest, Maintain1ChannelOnMediumUplinkBandwidth) { + constexpr int kInitChannels = 1; + auto controller = CreateChannelController(kInitChannels); + // Use between-thresholds bandwidth to check output remains at 1. + CheckDecision(controller.get(), kMediumBandwidthBps, 1); +} + +TEST(ChannelControllerTest, Maintain2ChannelsOnMediumUplinkBandwidth) { + constexpr int kInitChannels = 2; + auto controller = CreateChannelController(kInitChannels); + // Use between-thresholds bandwidth to check output remains at 2. + CheckDecision(controller.get(), kMediumBandwidthBps, 2); +} + +TEST(ChannelControllerTest, CheckBehaviorOnChangingUplinkBandwidth) { + constexpr int kInitChannels = 1; + auto controller = CreateChannelController(kInitChannels); + + // Use between-thresholds bandwidth to check output remains at 1. + CheckDecision(controller.get(), kMediumBandwidthBps, 1); + + // Use high bandwidth to check output switch to 2. + CheckDecision(controller.get(), kChannel1To2BandwidthBps, 2); + + // Use between-thresholds bandwidth to check output remains at 2. + CheckDecision(controller.get(), kMediumBandwidthBps, 2); + + // Use low bandwidth to check output switch to 1. + CheckDecision(controller.get(), kChannel2To1BandwidthBps, 1); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/config.proto b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/config.proto new file mode 100644 index 0000000000..a815451993 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/config.proto @@ -0,0 +1,196 @@ +syntax = "proto2"; + +package webrtc.audio_network_adaptor.config; + +option optimize_for = LITE_RUNTIME; +option java_package = "org.webrtc.AudioNetworkAdaptor"; +option java_outer_classname = "Config"; +option objc_class_prefix = "WANA"; + +message FecController { + message Threshold { + // Threshold defines a curve in the bandwidth/packet-loss domain. The + // curve is characterized by the two conjunction points: A and B. + // + // packet ^ | + // loss | A| + // | \ A: (low_bandwidth_bps, low_bandwidth_packet_loss) + // | \ B: (high_bandwidth_bps, high_bandwidth_packet_loss) + // | B\________ + // |---------------> bandwidth + optional int32 low_bandwidth_bps = 1; + optional float low_bandwidth_packet_loss = 2; + optional int32 high_bandwidth_bps = 3; + optional float high_bandwidth_packet_loss = 4; + } + + // `fec_enabling_threshold` defines a curve, above which FEC should be + // enabled. `fec_disabling_threshold` defines a curve, under which FEC + // should be disabled. See below + // + // packet-loss ^ | | + // | | | FEC + // | \ \ ON + // | FEC \ \_______ fec_enabling_threshold + // | OFF \_________ fec_disabling_threshold + // |-----------------> bandwidth + optional Threshold fec_enabling_threshold = 1; + optional Threshold fec_disabling_threshold = 2; + + // `time_constant_ms` is the time constant for an exponential filter, which + // is used for smoothing the packet loss fraction. + optional int32 time_constant_ms = 3; +} + +message FecControllerRplrBased { + message Threshold { + // Threshold defines a curve in the bandwidth/recoverable-packet-loss + // domain. + // The curve is characterized by the two conjunction points: A and B. + // + // recoverable ^ + // packet | | + // loss | A| + // | \ A: (low_bandwidth_bps, + // | \ low_bandwidth_recoverable_packet_loss) + // | \ B: (high_bandwidth_bps, + // | \ high_bandwidth_recoverable_packet_loss) + // | B\________ + // |---------------> bandwidth + optional int32 low_bandwidth_bps = 1; + optional float low_bandwidth_recoverable_packet_loss = 2; + optional int32 high_bandwidth_bps = 3; + optional float high_bandwidth_recoverable_packet_loss = 4; + } + + // `fec_enabling_threshold` defines a curve, above which FEC should be + // enabled. `fec_disabling_threshold` defines a curve, under which FEC + // should be disabled. See below + // + // packet-loss ^ | | + // | | | FEC + // | \ \ ON + // | FEC \ \_______ fec_enabling_threshold + // | OFF \_________ fec_disabling_threshold + // |-----------------> bandwidth + optional Threshold fec_enabling_threshold = 1; + optional Threshold fec_disabling_threshold = 2; +} + +message FrameLengthController { + // Uplink packet loss fraction below which frame length can increase. + optional float fl_increasing_packet_loss_fraction = 1; + + // Uplink packet loss fraction above which frame length should decrease. + optional float fl_decreasing_packet_loss_fraction = 2; + + // Uplink bandwidth below which frame length can switch from 20ms to 60ms. + optional int32 fl_20ms_to_60ms_bandwidth_bps = 3; + + // Uplink bandwidth above which frame length should switch from 60ms to 20ms. + optional int32 fl_60ms_to_20ms_bandwidth_bps = 4; + + // Uplink bandwidth below which frame length can switch from 60ms to 120ms. + optional int32 fl_60ms_to_120ms_bandwidth_bps = 5; + + // Uplink bandwidth above which frame length should switch from 120ms to 60ms. + optional int32 fl_120ms_to_60ms_bandwidth_bps = 6; + + // Offset to apply to the per-packet overhead when increasing frame length. + optional int32 fl_increase_overhead_offset = 7; + + // Offset to apply to the per-packet overhead when decreasing frame length. + optional int32 fl_decrease_overhead_offset = 8; + + // Uplink bandwidth below which frame length can switch from 20ms to 40ms. In + // current implementation, defining this will invalidate + // fl_20ms_to_60ms_bandwidth_bps. + optional int32 fl_20ms_to_40ms_bandwidth_bps = 9; + + // Uplink bandwidth above which frame length should switch from 40ms to 20ms. + optional int32 fl_40ms_to_20ms_bandwidth_bps = 10; + + // Uplink bandwidth below which frame length can switch from 40ms to 60ms. + optional int32 fl_40ms_to_60ms_bandwidth_bps = 11; + + // Uplink bandwidth above which frame length should switch from 60ms to 40ms. + // In current implementation, defining this will invalidate + // fl_60ms_to_20ms_bandwidth_bps. + optional int32 fl_60ms_to_40ms_bandwidth_bps = 12; +} + +message FrameLengthControllerV2 { + // FrameLengthControllerV2 chooses the frame length by taking the target + // bitrate and subtracting the overhead bitrate to obtain the remaining + // bitrate for the payload. The chosen frame length is the shortest possible + // where the payload bitrate is more than `min_payload_bitrate_bps`. + optional int32 min_payload_bitrate_bps = 1; + + // If true, uses the stable target bitrate to decide the frame length. This + // will result in less frame length toggling but spending more time at longer + // frame lengths compared to using the normal target bitrate. + optional bool use_slow_adaptation = 2; +} + +message ChannelController { + // Uplink bandwidth above which the number of encoded channels should switch + // from 1 to 2. + optional int32 channel_1_to_2_bandwidth_bps = 1; + + // Uplink bandwidth below which the number of encoded channels should switch + // from 2 to 1. + optional int32 channel_2_to_1_bandwidth_bps = 2; +} + +message DtxController { + // Uplink bandwidth below which DTX should be switched on. + optional int32 dtx_enabling_bandwidth_bps = 1; + + // Uplink bandwidth above which DTX should be switched off. + optional int32 dtx_disabling_bandwidth_bps = 2; +} + +message BitrateController { + // Offset to apply to per-packet overhead when the frame length is increased. + optional int32 fl_increase_overhead_offset = 1; + // Offset to apply to per-packet overhead when the frame length is decreased. + optional int32 fl_decrease_overhead_offset = 2; +} + +message Controller { + message ScoringPoint { + // `ScoringPoint` is a subspace of network condition. It is used for + // comparing the significance of controllers. + optional int32 uplink_bandwidth_bps = 1; + optional float uplink_packet_loss_fraction = 2; + } + + // The distance from `scoring_point` to a given network condition defines + // the significance of this controller with respect that network condition. + // Shorter distance means higher significance. The significances of + // controllers determine their order in the processing pipeline. Controllers + // without `scoring_point` follow their default order in + // `ControllerManager::controllers`. + optional ScoringPoint scoring_point = 1; + + oneof controller { + FecController fec_controller = 21; + FrameLengthController frame_length_controller = 22; + ChannelController channel_controller = 23; + DtxController dtx_controller = 24; + BitrateController bitrate_controller = 25; + FecControllerRplrBased fec_controller_rplr_based = 26; + FrameLengthControllerV2 frame_length_controller_v2 = 27; + } +} + +message ControllerManager { + repeated Controller controllers = 1; + + // Least time since last reordering for a new reordering to be made. + optional int32 min_reordering_time_ms = 2; + + // Least squared distance from last scoring point for a new reordering to be + // made. + optional float min_reordering_squared_distance = 3; +} diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.cc new file mode 100644 index 0000000000..5e2dc859bf --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.cc @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/controller.h" + +namespace webrtc { + +Controller::NetworkMetrics::NetworkMetrics() = default; + +Controller::NetworkMetrics::~NetworkMetrics() = default; + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.h new file mode 100644 index 0000000000..b70ada01a4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_H_ + +#include "absl/types/optional.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" + +namespace webrtc { + +class Controller { + public: + struct NetworkMetrics { + NetworkMetrics(); + ~NetworkMetrics(); + absl::optional uplink_bandwidth_bps; + absl::optional uplink_packet_loss_fraction; + absl::optional target_audio_bitrate_bps; + absl::optional rtt_ms; + absl::optional overhead_bytes_per_packet; + }; + + virtual ~Controller() = default; + + // Informs network metrics update to this controller. Any non-empty field + // indicates an update on the corresponding network metric. + virtual void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) = 0; + + virtual void MakeDecision(AudioEncoderRuntimeConfig* config) = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc new file mode 100644 index 0000000000..42dd8a8786 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc @@ -0,0 +1,454 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/audio_network_adaptor/bitrate_controller.h" +#include "modules/audio_coding/audio_network_adaptor/channel_controller.h" +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" +#include "modules/audio_coding/audio_network_adaptor/dtx_controller.h" +#include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h" +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h" +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" +#include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" + +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/config.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +namespace { + +#if WEBRTC_ENABLE_PROTOBUF + +std::unique_ptr CreateFecControllerPlrBased( + const audio_network_adaptor::config::FecController& config, + bool initial_fec_enabled) { + RTC_CHECK(config.has_fec_enabling_threshold()); + RTC_CHECK(config.has_fec_disabling_threshold()); + RTC_CHECK(config.has_time_constant_ms()); + + auto& fec_enabling_threshold = config.fec_enabling_threshold(); + RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_bps()); + RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_packet_loss()); + RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_bps()); + RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_packet_loss()); + + auto& fec_disabling_threshold = config.fec_disabling_threshold(); + RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_bps()); + RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_packet_loss()); + RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_bps()); + RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_packet_loss()); + + return std::unique_ptr( + new FecControllerPlrBased(FecControllerPlrBased::Config( + initial_fec_enabled, + ThresholdCurve(fec_enabling_threshold.low_bandwidth_bps(), + fec_enabling_threshold.low_bandwidth_packet_loss(), + fec_enabling_threshold.high_bandwidth_bps(), + fec_enabling_threshold.high_bandwidth_packet_loss()), + ThresholdCurve(fec_disabling_threshold.low_bandwidth_bps(), + fec_disabling_threshold.low_bandwidth_packet_loss(), + fec_disabling_threshold.high_bandwidth_bps(), + fec_disabling_threshold.high_bandwidth_packet_loss()), + config.time_constant_ms()))); +} + +std::unique_ptr CreateFrameLengthController( + const audio_network_adaptor::config::FrameLengthController& config, + rtc::ArrayView encoder_frame_lengths_ms, + int initial_frame_length_ms, + int min_encoder_bitrate_bps) { + RTC_CHECK(config.has_fl_increasing_packet_loss_fraction()); + RTC_CHECK(config.has_fl_decreasing_packet_loss_fraction()); + + std::map + fl_changing_bandwidths_bps; + + if (config.has_fl_20ms_to_60ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert( + std::make_pair(FrameLengthController::Config::FrameLengthChange(20, 60), + config.fl_20ms_to_60ms_bandwidth_bps())); + } + + if (config.has_fl_60ms_to_20ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert( + std::make_pair(FrameLengthController::Config::FrameLengthChange(60, 20), + config.fl_60ms_to_20ms_bandwidth_bps())); + } + + if (config.has_fl_20ms_to_40ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert( + std::make_pair(FrameLengthController::Config::FrameLengthChange(20, 40), + config.fl_20ms_to_40ms_bandwidth_bps())); + } + + if (config.has_fl_40ms_to_20ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert( + std::make_pair(FrameLengthController::Config::FrameLengthChange(40, 20), + config.fl_40ms_to_20ms_bandwidth_bps())); + } + + if (config.has_fl_40ms_to_60ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert( + std::make_pair(FrameLengthController::Config::FrameLengthChange(40, 60), + config.fl_40ms_to_60ms_bandwidth_bps())); + } + + if (config.has_fl_60ms_to_40ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert( + std::make_pair(FrameLengthController::Config::FrameLengthChange(60, 40), + config.fl_60ms_to_40ms_bandwidth_bps())); + } + + if (config.has_fl_60ms_to_120ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert(std::make_pair( + FrameLengthController::Config::FrameLengthChange(60, 120), + config.fl_60ms_to_120ms_bandwidth_bps())); + } + + if (config.has_fl_120ms_to_60ms_bandwidth_bps()) { + fl_changing_bandwidths_bps.insert(std::make_pair( + FrameLengthController::Config::FrameLengthChange(120, 60), + config.fl_120ms_to_60ms_bandwidth_bps())); + } + + int fl_increase_overhead_offset = 0; + if (config.has_fl_increase_overhead_offset()) { + fl_increase_overhead_offset = config.fl_increase_overhead_offset(); + } + int fl_decrease_overhead_offset = 0; + if (config.has_fl_decrease_overhead_offset()) { + fl_decrease_overhead_offset = config.fl_decrease_overhead_offset(); + } + + FrameLengthController::Config ctor_config( + std::set(), initial_frame_length_ms, min_encoder_bitrate_bps, + config.fl_increasing_packet_loss_fraction(), + config.fl_decreasing_packet_loss_fraction(), fl_increase_overhead_offset, + fl_decrease_overhead_offset, std::move(fl_changing_bandwidths_bps)); + + for (auto frame_length : encoder_frame_lengths_ms) + ctor_config.encoder_frame_lengths_ms.insert(frame_length); + + return std::unique_ptr( + new FrameLengthController(ctor_config)); +} + +std::unique_ptr CreateChannelController( + const audio_network_adaptor::config::ChannelController& config, + size_t num_encoder_channels, + size_t intial_channels_to_encode) { + RTC_CHECK(config.has_channel_1_to_2_bandwidth_bps()); + RTC_CHECK(config.has_channel_2_to_1_bandwidth_bps()); + + return std::unique_ptr(new ChannelController( + ChannelController::Config(num_encoder_channels, intial_channels_to_encode, + config.channel_1_to_2_bandwidth_bps(), + config.channel_2_to_1_bandwidth_bps()))); +} + +std::unique_ptr CreateDtxController( + const audio_network_adaptor::config::DtxController& dtx_config, + bool initial_dtx_enabled) { + RTC_CHECK(dtx_config.has_dtx_enabling_bandwidth_bps()); + RTC_CHECK(dtx_config.has_dtx_disabling_bandwidth_bps()); + + return std::unique_ptr(new DtxController(DtxController::Config( + initial_dtx_enabled, dtx_config.dtx_enabling_bandwidth_bps(), + dtx_config.dtx_disabling_bandwidth_bps()))); +} + +using audio_network_adaptor::BitrateController; +std::unique_ptr CreateBitrateController( + const audio_network_adaptor::config::BitrateController& bitrate_config, + int initial_bitrate_bps, + int initial_frame_length_ms) { + int fl_increase_overhead_offset = 0; + if (bitrate_config.has_fl_increase_overhead_offset()) { + fl_increase_overhead_offset = bitrate_config.fl_increase_overhead_offset(); + } + int fl_decrease_overhead_offset = 0; + if (bitrate_config.has_fl_decrease_overhead_offset()) { + fl_decrease_overhead_offset = bitrate_config.fl_decrease_overhead_offset(); + } + return std::unique_ptr( + new BitrateController(BitrateController::Config( + initial_bitrate_bps, initial_frame_length_ms, + fl_increase_overhead_offset, fl_decrease_overhead_offset))); +} + +std::unique_ptr CreateFrameLengthControllerV2( + const audio_network_adaptor::config::FrameLengthControllerV2& config, + rtc::ArrayView encoder_frame_lengths_ms) { + return std::make_unique( + encoder_frame_lengths_ms, config.min_payload_bitrate_bps(), + config.use_slow_adaptation()); +} +#endif // WEBRTC_ENABLE_PROTOBUF + +} // namespace + +ControllerManagerImpl::Config::Config(int min_reordering_time_ms, + float min_reordering_squared_distance) + : min_reordering_time_ms(min_reordering_time_ms), + min_reordering_squared_distance(min_reordering_squared_distance) {} + +ControllerManagerImpl::Config::~Config() = default; + +std::unique_ptr ControllerManagerImpl::Create( + absl::string_view config_string, + size_t num_encoder_channels, + rtc::ArrayView encoder_frame_lengths_ms, + int min_encoder_bitrate_bps, + size_t intial_channels_to_encode, + int initial_frame_length_ms, + int initial_bitrate_bps, + bool initial_fec_enabled, + bool initial_dtx_enabled) { + return Create(config_string, num_encoder_channels, encoder_frame_lengths_ms, + min_encoder_bitrate_bps, intial_channels_to_encode, + initial_frame_length_ms, initial_bitrate_bps, + initial_fec_enabled, initial_dtx_enabled, nullptr); +} + +std::unique_ptr ControllerManagerImpl::Create( + absl::string_view config_string, + size_t num_encoder_channels, + rtc::ArrayView encoder_frame_lengths_ms, + int min_encoder_bitrate_bps, + size_t intial_channels_to_encode, + int initial_frame_length_ms, + int initial_bitrate_bps, + bool initial_fec_enabled, + bool initial_dtx_enabled, + DebugDumpWriter* debug_dump_writer) { +#if WEBRTC_ENABLE_PROTOBUF + audio_network_adaptor::config::ControllerManager controller_manager_config; + RTC_CHECK( + controller_manager_config.ParseFromString(std::string(config_string))); + if (debug_dump_writer) + debug_dump_writer->DumpControllerManagerConfig(controller_manager_config, + rtc::TimeMillis()); + + std::vector> controllers; + std::map> scoring_points; + + for (int i = 0; i < controller_manager_config.controllers_size(); ++i) { + auto& controller_config = controller_manager_config.controllers(i); + std::unique_ptr controller; + switch (controller_config.controller_case()) { + case audio_network_adaptor::config::Controller::kFecController: + controller = CreateFecControllerPlrBased( + controller_config.fec_controller(), initial_fec_enabled); + break; + case audio_network_adaptor::config::Controller::kFecControllerRplrBased: + // FecControllerRplrBased has been removed and can't be used anymore. + RTC_DCHECK_NOTREACHED(); + continue; + case audio_network_adaptor::config::Controller::kFrameLengthController: + controller = CreateFrameLengthController( + controller_config.frame_length_controller(), + encoder_frame_lengths_ms, initial_frame_length_ms, + min_encoder_bitrate_bps); + break; + case audio_network_adaptor::config::Controller::kChannelController: + controller = CreateChannelController( + controller_config.channel_controller(), num_encoder_channels, + intial_channels_to_encode); + break; + case audio_network_adaptor::config::Controller::kDtxController: + controller = CreateDtxController(controller_config.dtx_controller(), + initial_dtx_enabled); + break; + case audio_network_adaptor::config::Controller::kBitrateController: + controller = CreateBitrateController( + controller_config.bitrate_controller(), initial_bitrate_bps, + initial_frame_length_ms); + break; + case audio_network_adaptor::config::Controller::kFrameLengthControllerV2: + controller = CreateFrameLengthControllerV2( + controller_config.frame_length_controller_v2(), + encoder_frame_lengths_ms); + break; + default: + RTC_DCHECK_NOTREACHED(); + } + if (controller_config.has_scoring_point()) { + auto& scoring_point = controller_config.scoring_point(); + RTC_CHECK(scoring_point.has_uplink_bandwidth_bps()); + RTC_CHECK(scoring_point.has_uplink_packet_loss_fraction()); + scoring_points[controller.get()] = std::make_pair( + scoring_point.uplink_bandwidth_bps(), + scoring_point.uplink_packet_loss_fraction()); + } + controllers.push_back(std::move(controller)); + } + + if (scoring_points.size() == 0) { + return std::unique_ptr( + new ControllerManagerImpl(ControllerManagerImpl::Config(0, 0), + std::move(controllers), scoring_points)); + } else { + RTC_CHECK(controller_manager_config.has_min_reordering_time_ms()); + RTC_CHECK(controller_manager_config.has_min_reordering_squared_distance()); + return std::unique_ptr(new ControllerManagerImpl( + ControllerManagerImpl::Config( + controller_manager_config.min_reordering_time_ms(), + controller_manager_config.min_reordering_squared_distance()), + std::move(controllers), scoring_points)); + } + +#else + RTC_DCHECK_NOTREACHED(); + return nullptr; +#endif // WEBRTC_ENABLE_PROTOBUF +} + +ControllerManagerImpl::ControllerManagerImpl(const Config& config) + : ControllerManagerImpl( + config, + std::vector>(), + std::map>()) {} + +ControllerManagerImpl::ControllerManagerImpl( + const Config& config, + std::vector> controllers, + const std::map>& scoring_points) + : config_(config), + controllers_(std::move(controllers)), + last_reordering_time_ms_(absl::nullopt), + last_scoring_point_(0, 0.0) { + for (auto& controller : controllers_) + default_sorted_controllers_.push_back(controller.get()); + sorted_controllers_ = default_sorted_controllers_; + for (auto& controller_point : scoring_points) { + controller_scoring_points_.insert(std::make_pair( + controller_point.first, ScoringPoint(controller_point.second.first, + controller_point.second.second))); + } +} + +ControllerManagerImpl::~ControllerManagerImpl() = default; + +std::vector ControllerManagerImpl::GetSortedControllers( + const Controller::NetworkMetrics& metrics) { + if (controller_scoring_points_.size() == 0) + return default_sorted_controllers_; + + if (!metrics.uplink_bandwidth_bps || !metrics.uplink_packet_loss_fraction) + return sorted_controllers_; + + const int64_t now_ms = rtc::TimeMillis(); + if (last_reordering_time_ms_ && + now_ms - *last_reordering_time_ms_ < config_.min_reordering_time_ms) + return sorted_controllers_; + + ScoringPoint scoring_point(*metrics.uplink_bandwidth_bps, + *metrics.uplink_packet_loss_fraction); + + if (last_reordering_time_ms_ && + last_scoring_point_.SquaredDistanceTo(scoring_point) < + config_.min_reordering_squared_distance) + return sorted_controllers_; + + // Sort controllers according to the distances of `scoring_point` to the + // scoring points of controllers. + // + // A controller that does not associate with any scoring point + // are treated as if + // 1) they are less important than any controller that has a scoring point, + // 2) they are equally important to any controller that has no scoring point, + // and their relative order will follow `default_sorted_controllers_`. + std::vector sorted_controllers(default_sorted_controllers_); + std::stable_sort( + sorted_controllers.begin(), sorted_controllers.end(), + [this, &scoring_point](const Controller* lhs, const Controller* rhs) { + auto lhs_scoring_point = controller_scoring_points_.find(lhs); + auto rhs_scoring_point = controller_scoring_points_.find(rhs); + + if (lhs_scoring_point == controller_scoring_points_.end()) + return false; + + if (rhs_scoring_point == controller_scoring_points_.end()) + return true; + + return lhs_scoring_point->second.SquaredDistanceTo(scoring_point) < + rhs_scoring_point->second.SquaredDistanceTo(scoring_point); + }); + + if (sorted_controllers_ != sorted_controllers) { + sorted_controllers_ = sorted_controllers; + last_reordering_time_ms_ = now_ms; + last_scoring_point_ = scoring_point; + } + return sorted_controllers_; +} + +std::vector ControllerManagerImpl::GetControllers() const { + return default_sorted_controllers_; +} + +ControllerManagerImpl::ScoringPoint::ScoringPoint( + int uplink_bandwidth_bps, + float uplink_packet_loss_fraction) + : uplink_bandwidth_bps(uplink_bandwidth_bps), + uplink_packet_loss_fraction(uplink_packet_loss_fraction) {} + +namespace { + +constexpr int kMinUplinkBandwidthBps = 0; +constexpr int kMaxUplinkBandwidthBps = 120000; + +float NormalizeUplinkBandwidth(int uplink_bandwidth_bps) { + uplink_bandwidth_bps = + std::min(kMaxUplinkBandwidthBps, + std::max(kMinUplinkBandwidthBps, uplink_bandwidth_bps)); + return static_cast(uplink_bandwidth_bps - kMinUplinkBandwidthBps) / + (kMaxUplinkBandwidthBps - kMinUplinkBandwidthBps); +} + +float NormalizePacketLossFraction(float uplink_packet_loss_fraction) { + // `uplink_packet_loss_fraction` is seldom larger than 0.3, so we scale it up + // by 3.3333f. + return std::min(uplink_packet_loss_fraction * 3.3333f, 1.0f); +} + +} // namespace + +float ControllerManagerImpl::ScoringPoint::SquaredDistanceTo( + const ScoringPoint& scoring_point) const { + float diff_normalized_bitrate_bps = + NormalizeUplinkBandwidth(scoring_point.uplink_bandwidth_bps) - + NormalizeUplinkBandwidth(uplink_bandwidth_bps); + float diff_normalized_packet_loss = + NormalizePacketLossFraction(scoring_point.uplink_packet_loss_fraction) - + NormalizePacketLossFraction(uplink_packet_loss_fraction); + return std::pow(diff_normalized_bitrate_bps, 2) + + std::pow(diff_normalized_packet_loss, 2); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h new file mode 100644 index 0000000000..47e8e0f5a0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_MANAGER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_MANAGER_H_ + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" + +namespace webrtc { + +class DebugDumpWriter; + +class ControllerManager { + public: + virtual ~ControllerManager() = default; + + // Sort controllers based on their significance. + virtual std::vector GetSortedControllers( + const Controller::NetworkMetrics& metrics) = 0; + + virtual std::vector GetControllers() const = 0; +}; + +class ControllerManagerImpl final : public ControllerManager { + public: + struct Config { + Config(int min_reordering_time_ms, float min_reordering_squared_distance); + ~Config(); + // Least time since last reordering for a new reordering to be made. + int min_reordering_time_ms; + // Least squared distance from last scoring point for a new reordering to be + // made. + float min_reordering_squared_distance; + }; + + static std::unique_ptr Create( + absl::string_view config_string, + size_t num_encoder_channels, + rtc::ArrayView encoder_frame_lengths_ms, + int min_encoder_bitrate_bps, + size_t intial_channels_to_encode, + int initial_frame_length_ms, + int initial_bitrate_bps, + bool initial_fec_enabled, + bool initial_dtx_enabled); + + static std::unique_ptr Create( + absl::string_view config_string, + size_t num_encoder_channels, + rtc::ArrayView encoder_frame_lengths_ms, + int min_encoder_bitrate_bps, + size_t intial_channels_to_encode, + int initial_frame_length_ms, + int initial_bitrate_bps, + bool initial_fec_enabled, + bool initial_dtx_enabled, + DebugDumpWriter* debug_dump_writer); + + explicit ControllerManagerImpl(const Config& config); + + // Dependency injection for testing. + ControllerManagerImpl( + const Config& config, + std::vector> controllers, + const std::map>& + chracteristic_points); + + ~ControllerManagerImpl() override; + + ControllerManagerImpl(const ControllerManagerImpl&) = delete; + ControllerManagerImpl& operator=(const ControllerManagerImpl&) = delete; + + // Sort controllers based on their significance. + std::vector GetSortedControllers( + const Controller::NetworkMetrics& metrics) override; + + std::vector GetControllers() const override; + + private: + // Scoring point is a subset of NetworkMetrics that is used for comparing the + // significance of controllers. + struct ScoringPoint { + // TODO(eladalon): Do we want to experiment with RPLR-based scoring? + ScoringPoint(int uplink_bandwidth_bps, float uplink_packet_loss_fraction); + + // Calculate the normalized [0,1] distance between two scoring points. + float SquaredDistanceTo(const ScoringPoint& scoring_point) const; + + int uplink_bandwidth_bps; + float uplink_packet_loss_fraction; + }; + + const Config config_; + + std::vector> controllers_; + + absl::optional last_reordering_time_ms_; + ScoringPoint last_scoring_point_; + + std::vector default_sorted_controllers_; + + std::vector sorted_controllers_; + + // `scoring_points_` saves the scoring points of various + // controllers. + std::map controller_scoring_points_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_CONTROLLER_MANAGER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc new file mode 100644 index 0000000000..3e6ecf6def --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc @@ -0,0 +1,486 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" + +#include +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_controller.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/ignore_wundef.h" +#include "test/gtest.h" + +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/config.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +using ::testing::_; +using ::testing::NiceMock; + +namespace { + +constexpr size_t kNumControllers = 4; +constexpr int kChracteristicBandwithBps[2] = {15000, 0}; +constexpr float kChracteristicPacketLossFraction[2] = {0.2f, 0.0f}; +constexpr int kMinReorderingTimeMs = 200; +constexpr int kFactor = 100; +constexpr float kMinReorderingSquareDistance = 1.0f / kFactor / kFactor; + +// `kMinUplinkBandwidthBps` and `kMaxUplinkBandwidthBps` are copied from +// controller_manager.cc +constexpr int kMinUplinkBandwidthBps = 0; +constexpr int kMaxUplinkBandwidthBps = 120000; +constexpr int kMinBandwithChangeBps = + (kMaxUplinkBandwidthBps - kMinUplinkBandwidthBps) / kFactor; + +struct ControllerManagerStates { + std::unique_ptr controller_manager; + std::vector mock_controllers; +}; + +ControllerManagerStates CreateControllerManager() { + ControllerManagerStates states; + std::vector> controllers; + std::map> chracteristic_points; + for (size_t i = 0; i < kNumControllers; ++i) { + auto controller = + std::unique_ptr(new NiceMock()); + EXPECT_CALL(*controller, Die()); + states.mock_controllers.push_back(controller.get()); + controllers.push_back(std::move(controller)); + } + + // Assign characteristic points to the last two controllers. + chracteristic_points[states.mock_controllers[kNumControllers - 2]] = + std::make_pair(kChracteristicBandwithBps[0], + kChracteristicPacketLossFraction[0]); + chracteristic_points[states.mock_controllers[kNumControllers - 1]] = + std::make_pair(kChracteristicBandwithBps[1], + kChracteristicPacketLossFraction[1]); + + states.controller_manager.reset(new ControllerManagerImpl( + ControllerManagerImpl::Config(kMinReorderingTimeMs, + kMinReorderingSquareDistance), + std::move(controllers), chracteristic_points)); + return states; +} + +// `expected_order` contains the expected indices of all controllers in the +// vector of controllers returned by GetSortedControllers(). A negative index +// means that we do not care about its exact place, but we do check that it +// exists in the vector. +void CheckControllersOrder( + ControllerManagerStates* states, + const absl::optional& uplink_bandwidth_bps, + const absl::optional& uplink_packet_loss_fraction, + const std::vector& expected_order) { + RTC_DCHECK_EQ(kNumControllers, expected_order.size()); + Controller::NetworkMetrics metrics; + metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + metrics.uplink_packet_loss_fraction = uplink_packet_loss_fraction; + auto check = states->controller_manager->GetSortedControllers(metrics); + EXPECT_EQ(states->mock_controllers.size(), check.size()); + for (size_t i = 0; i < states->mock_controllers.size(); ++i) { + if (expected_order[i] >= 0) { + EXPECT_EQ(states->mock_controllers[i], check[expected_order[i]]); + } else { + EXPECT_NE(check.end(), std::find(check.begin(), check.end(), + states->mock_controllers[i])); + } + } +} + +} // namespace + +TEST(ControllerManagerTest, GetControllersReturnAllControllers) { + auto states = CreateControllerManager(); + auto check = states.controller_manager->GetControllers(); + // Verify that controllers in `check` are one-to-one mapped to those in + // `mock_controllers_`. + EXPECT_EQ(states.mock_controllers.size(), check.size()); + for (auto& controller : check) + EXPECT_NE(states.mock_controllers.end(), + std::find(states.mock_controllers.begin(), + states.mock_controllers.end(), controller)); +} + +TEST(ControllerManagerTest, ControllersInDefaultOrderOnEmptyNetworkMetrics) { + auto states = CreateControllerManager(); + // `network_metrics` are empty, and the controllers are supposed to follow the + // default order. + CheckControllersOrder(&states, absl::nullopt, absl::nullopt, {0, 1, 2, 3}); +} + +TEST(ControllerManagerTest, ControllersWithoutCharPointAtEndAndInDefaultOrder) { + auto states = CreateControllerManager(); + CheckControllersOrder(&states, 0, 0.0, + {kNumControllers - 2, kNumControllers - 1, -1, -1}); +} + +TEST(ControllerManagerTest, ControllersWithCharPointDependOnNetworkMetrics) { + auto states = CreateControllerManager(); + CheckControllersOrder(&states, kChracteristicBandwithBps[1], + kChracteristicPacketLossFraction[1], + {kNumControllers - 2, kNumControllers - 1, 1, 0}); +} + +TEST(ControllerManagerTest, DoNotReorderBeforeMinReordingTime) { + rtc::ScopedFakeClock fake_clock; + auto states = CreateControllerManager(); + CheckControllersOrder(&states, kChracteristicBandwithBps[0], + kChracteristicPacketLossFraction[0], + {kNumControllers - 2, kNumControllers - 1, 0, 1}); + fake_clock.AdvanceTime(TimeDelta::Millis(kMinReorderingTimeMs - 1)); + // Move uplink bandwidth and packet loss fraction to the other controller's + // characteristic point, which would cause controller manager to reorder the + // controllers if time had reached min reordering time. + CheckControllersOrder(&states, kChracteristicBandwithBps[1], + kChracteristicPacketLossFraction[1], + {kNumControllers - 2, kNumControllers - 1, 0, 1}); +} + +TEST(ControllerManagerTest, ReorderBeyondMinReordingTimeAndMinDistance) { + rtc::ScopedFakeClock fake_clock; + auto states = CreateControllerManager(); + constexpr int kBandwidthBps = + (kChracteristicBandwithBps[0] + kChracteristicBandwithBps[1]) / 2; + constexpr float kPacketLossFraction = (kChracteristicPacketLossFraction[0] + + kChracteristicPacketLossFraction[1]) / + 2.0f; + // Set network metrics to be in the middle between the characteristic points + // of two controllers. + CheckControllersOrder(&states, kBandwidthBps, kPacketLossFraction, + {kNumControllers - 2, kNumControllers - 1, 0, 1}); + fake_clock.AdvanceTime(TimeDelta::Millis(kMinReorderingTimeMs)); + // Then let network metrics move a little towards the other controller. + CheckControllersOrder(&states, kBandwidthBps - kMinBandwithChangeBps - 1, + kPacketLossFraction, + {kNumControllers - 2, kNumControllers - 1, 1, 0}); +} + +TEST(ControllerManagerTest, DoNotReorderIfNetworkMetricsChangeTooSmall) { + rtc::ScopedFakeClock fake_clock; + auto states = CreateControllerManager(); + constexpr int kBandwidthBps = + (kChracteristicBandwithBps[0] + kChracteristicBandwithBps[1]) / 2; + constexpr float kPacketLossFraction = (kChracteristicPacketLossFraction[0] + + kChracteristicPacketLossFraction[1]) / + 2.0f; + // Set network metrics to be in the middle between the characteristic points + // of two controllers. + CheckControllersOrder(&states, kBandwidthBps, kPacketLossFraction, + {kNumControllers - 2, kNumControllers - 1, 0, 1}); + fake_clock.AdvanceTime(TimeDelta::Millis(kMinReorderingTimeMs)); + // Then let network metrics move a little towards the other controller. + CheckControllersOrder(&states, kBandwidthBps - kMinBandwithChangeBps + 1, + kPacketLossFraction, + {kNumControllers - 2, kNumControllers - 1, 0, 1}); +} + +#if WEBRTC_ENABLE_PROTOBUF + +namespace { + +void AddBitrateControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + config->add_controllers()->mutable_bitrate_controller(); +} + +void AddChannelControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + auto controller_config = + config->add_controllers()->mutable_channel_controller(); + controller_config->set_channel_1_to_2_bandwidth_bps(31000); + controller_config->set_channel_2_to_1_bandwidth_bps(29000); +} + +void AddDtxControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + auto controller_config = config->add_controllers()->mutable_dtx_controller(); + controller_config->set_dtx_enabling_bandwidth_bps(55000); + controller_config->set_dtx_disabling_bandwidth_bps(65000); +} + +void AddFecControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + auto controller_config_ext = config->add_controllers(); + auto controller_config = controller_config_ext->mutable_fec_controller(); + auto fec_enabling_threshold = + controller_config->mutable_fec_enabling_threshold(); + fec_enabling_threshold->set_low_bandwidth_bps(17000); + fec_enabling_threshold->set_low_bandwidth_packet_loss(0.1f); + fec_enabling_threshold->set_high_bandwidth_bps(64000); + fec_enabling_threshold->set_high_bandwidth_packet_loss(0.05f); + auto fec_disabling_threshold = + controller_config->mutable_fec_disabling_threshold(); + fec_disabling_threshold->set_low_bandwidth_bps(15000); + fec_disabling_threshold->set_low_bandwidth_packet_loss(0.08f); + fec_disabling_threshold->set_high_bandwidth_bps(64000); + fec_disabling_threshold->set_high_bandwidth_packet_loss(0.01f); + controller_config->set_time_constant_ms(500); + + auto scoring_point = controller_config_ext->mutable_scoring_point(); + scoring_point->set_uplink_bandwidth_bps(kChracteristicBandwithBps[0]); + scoring_point->set_uplink_packet_loss_fraction( + kChracteristicPacketLossFraction[0]); +} + +void AddFrameLengthControllerConfig( + audio_network_adaptor::config::ControllerManager* config) { + auto controller_config_ext = config->add_controllers(); + auto controller_config = + controller_config_ext->mutable_frame_length_controller(); + controller_config->set_fl_decreasing_packet_loss_fraction(0.05f); + controller_config->set_fl_increasing_packet_loss_fraction(0.04f); + controller_config->set_fl_20ms_to_40ms_bandwidth_bps(80000); + controller_config->set_fl_40ms_to_20ms_bandwidth_bps(88000); + controller_config->set_fl_40ms_to_60ms_bandwidth_bps(72000); + controller_config->set_fl_60ms_to_40ms_bandwidth_bps(80000); + + auto scoring_point = controller_config_ext->mutable_scoring_point(); + scoring_point->set_uplink_bandwidth_bps(kChracteristicBandwithBps[1]); + scoring_point->set_uplink_packet_loss_fraction( + kChracteristicPacketLossFraction[1]); +} + +void AddFrameLengthControllerV2Config( + audio_network_adaptor::config::ControllerManager* config) { + auto controller = + config->add_controllers()->mutable_frame_length_controller_v2(); + controller->set_min_payload_bitrate_bps(16000); + controller->set_use_slow_adaptation(true); +} + +constexpr int kInitialBitrateBps = 24000; +constexpr size_t kIntialChannelsToEncode = 1; +constexpr bool kInitialDtxEnabled = true; +constexpr bool kInitialFecEnabled = true; +constexpr int kInitialFrameLengthMs = 60; +constexpr int kMinBitrateBps = 6000; + +ControllerManagerStates CreateControllerManager( + absl::string_view config_string) { + ControllerManagerStates states; + constexpr size_t kNumEncoderChannels = 2; + const std::vector encoder_frame_lengths_ms = {20, 60}; + states.controller_manager = ControllerManagerImpl::Create( + config_string, kNumEncoderChannels, encoder_frame_lengths_ms, + kMinBitrateBps, kIntialChannelsToEncode, kInitialFrameLengthMs, + kInitialBitrateBps, kInitialFecEnabled, kInitialDtxEnabled); + return states; +} + +enum class ControllerType : int8_t { + FEC, + CHANNEL, + DTX, + FRAME_LENGTH, + BIT_RATE +}; + +void CheckControllersOrder(const std::vector& controllers, + const std::vector& expected_types) { + ASSERT_EQ(expected_types.size(), controllers.size()); + + // We also check that the controllers follow the initial settings. + AudioEncoderRuntimeConfig encoder_config; + + for (size_t i = 0; i < controllers.size(); ++i) { + AudioEncoderRuntimeConfig encoder_config; + // We check the order of `controllers` by judging their decisions. + controllers[i]->MakeDecision(&encoder_config); + + // Since controllers are not provided with network metrics, they give the + // initial values. + switch (expected_types[i]) { + case ControllerType::FEC: + EXPECT_EQ(kInitialFecEnabled, encoder_config.enable_fec); + break; + case ControllerType::CHANNEL: + EXPECT_EQ(kIntialChannelsToEncode, encoder_config.num_channels); + break; + case ControllerType::DTX: + EXPECT_EQ(kInitialDtxEnabled, encoder_config.enable_dtx); + break; + case ControllerType::FRAME_LENGTH: + EXPECT_EQ(kInitialFrameLengthMs, encoder_config.frame_length_ms); + break; + case ControllerType::BIT_RATE: + EXPECT_EQ(kInitialBitrateBps, encoder_config.bitrate_bps); + } + } +} + +MATCHER_P(ControllerManagerEqual, value, "") { + std::string value_string; + std::string arg_string; + EXPECT_TRUE(arg.SerializeToString(&arg_string)); + EXPECT_TRUE(value.SerializeToString(&value_string)); + return arg_string == value_string; +} + +} // namespace + +TEST(ControllerManagerTest, DebugDumpLoggedWhenCreateFromConfigString) { + audio_network_adaptor::config::ControllerManager config; + config.set_min_reordering_time_ms(kMinReorderingTimeMs); + config.set_min_reordering_squared_distance(kMinReorderingSquareDistance); + + AddFecControllerConfig(&config); + AddChannelControllerConfig(&config); + AddDtxControllerConfig(&config); + AddFrameLengthControllerConfig(&config); + AddBitrateControllerConfig(&config); + + std::string config_string; + config.SerializeToString(&config_string); + + constexpr size_t kNumEncoderChannels = 2; + const std::vector encoder_frame_lengths_ms = {20, 60}; + + constexpr int64_t kClockInitialTimeMs = 12345678; + rtc::ScopedFakeClock fake_clock; + fake_clock.AdvanceTime(TimeDelta::Millis(kClockInitialTimeMs)); + auto debug_dump_writer = + std::unique_ptr(new NiceMock()); + EXPECT_CALL(*debug_dump_writer, Die()); + EXPECT_CALL(*debug_dump_writer, + DumpControllerManagerConfig(ControllerManagerEqual(config), + kClockInitialTimeMs)); + + ControllerManagerImpl::Create(config_string, kNumEncoderChannels, + encoder_frame_lengths_ms, kMinBitrateBps, + kIntialChannelsToEncode, kInitialFrameLengthMs, + kInitialBitrateBps, kInitialFecEnabled, + kInitialDtxEnabled, debug_dump_writer.get()); +} + +TEST(ControllerManagerTest, CreateFromConfigStringAndCheckDefaultOrder) { + audio_network_adaptor::config::ControllerManager config; + config.set_min_reordering_time_ms(kMinReorderingTimeMs); + config.set_min_reordering_squared_distance(kMinReorderingSquareDistance); + + AddFecControllerConfig(&config); + AddChannelControllerConfig(&config); + AddDtxControllerConfig(&config); + AddFrameLengthControllerConfig(&config); + AddBitrateControllerConfig(&config); + + std::string config_string; + config.SerializeToString(&config_string); + + auto states = CreateControllerManager(config_string); + Controller::NetworkMetrics metrics; + + auto controllers = states.controller_manager->GetSortedControllers(metrics); + CheckControllersOrder( + controllers, + std::vector{ + ControllerType::FEC, ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::FRAME_LENGTH, ControllerType::BIT_RATE}); +} + +TEST(ControllerManagerTest, CreateCharPointFreeConfigAndCheckDefaultOrder) { + audio_network_adaptor::config::ControllerManager config; + + // Following controllers have no characteristic points. + AddChannelControllerConfig(&config); + AddDtxControllerConfig(&config); + AddBitrateControllerConfig(&config); + + std::string config_string; + config.SerializeToString(&config_string); + + auto states = CreateControllerManager(config_string); + Controller::NetworkMetrics metrics; + + auto controllers = states.controller_manager->GetSortedControllers(metrics); + CheckControllersOrder( + controllers, + std::vector{ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::BIT_RATE}); +} + +TEST(ControllerManagerTest, CreateFromConfigStringAndCheckReordering) { + rtc::ScopedFakeClock fake_clock; + audio_network_adaptor::config::ControllerManager config; + config.set_min_reordering_time_ms(kMinReorderingTimeMs); + config.set_min_reordering_squared_distance(kMinReorderingSquareDistance); + + AddChannelControllerConfig(&config); + + // Internally associated with characteristic point 0. + AddFecControllerConfig(&config); + + AddDtxControllerConfig(&config); + + // Internally associated with characteristic point 1. + AddFrameLengthControllerConfig(&config); + + AddBitrateControllerConfig(&config); + + std::string config_string; + config.SerializeToString(&config_string); + + auto states = CreateControllerManager(config_string); + + Controller::NetworkMetrics metrics; + metrics.uplink_bandwidth_bps = kChracteristicBandwithBps[0]; + metrics.uplink_packet_loss_fraction = kChracteristicPacketLossFraction[0]; + + auto controllers = states.controller_manager->GetSortedControllers(metrics); + CheckControllersOrder(controllers, + std::vector{ + ControllerType::FEC, ControllerType::FRAME_LENGTH, + ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::BIT_RATE}); + + metrics.uplink_bandwidth_bps = kChracteristicBandwithBps[1]; + metrics.uplink_packet_loss_fraction = kChracteristicPacketLossFraction[1]; + fake_clock.AdvanceTime(TimeDelta::Millis(kMinReorderingTimeMs - 1)); + controllers = states.controller_manager->GetSortedControllers(metrics); + // Should not reorder since min reordering time is not met. + CheckControllersOrder(controllers, + std::vector{ + ControllerType::FEC, ControllerType::FRAME_LENGTH, + ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::BIT_RATE}); + + fake_clock.AdvanceTime(TimeDelta::Millis(1)); + controllers = states.controller_manager->GetSortedControllers(metrics); + // Reorder now. + CheckControllersOrder(controllers, + std::vector{ + ControllerType::FRAME_LENGTH, ControllerType::FEC, + ControllerType::CHANNEL, ControllerType::DTX, + ControllerType::BIT_RATE}); +} + +TEST(ControllerManagerTest, CreateFrameLengthControllerV2) { + audio_network_adaptor::config::ControllerManager config; + AddFrameLengthControllerV2Config(&config); + auto states = CreateControllerManager(config.SerializeAsString()); + auto controllers = states.controller_manager->GetControllers(); + EXPECT_TRUE(controllers.size() == 1); +} +#endif // WEBRTC_ENABLE_PROTOBUF + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto new file mode 100644 index 0000000000..3aa6a504f3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto @@ -0,0 +1,42 @@ +syntax = "proto2"; +option optimize_for = LITE_RUNTIME; +package webrtc.audio_network_adaptor.debug_dump; + +import "config.proto"; + +message NetworkMetrics { + optional int32 uplink_bandwidth_bps = 1; + optional float uplink_packet_loss_fraction = 2; + optional int32 target_audio_bitrate_bps = 3; + optional int32 rtt_ms = 4; + optional int32 uplink_recoverable_packet_loss_fraction = 5; +} + +message EncoderRuntimeConfig { + optional int32 bitrate_bps = 1; + optional int32 frame_length_ms = 2; + // Note: This is what we tell the encoder. It doesn't have to reflect + // the actual NetworkMetrics; it's subject to our decision. + optional float uplink_packet_loss_fraction = 3; + optional bool enable_fec = 4; + optional bool enable_dtx = 5; + // Some encoders can encode fewer channels than the actual input to make + // better use of the bandwidth. `num_channels` sets the number of channels + // to encode. + optional uint32 num_channels = 6; +} + +message Event { + enum Type { + NETWORK_METRICS = 0; + ENCODER_RUNTIME_CONFIG = 1; + CONTROLLER_MANAGER_CONFIG = 2; + } + required Type type = 1; + required uint32 timestamp = 2; + optional NetworkMetrics network_metrics = 3; + optional EncoderRuntimeConfig encoder_runtime_config = 4; + optional webrtc.audio_network_adaptor.config.ControllerManager + controller_manager_config = 5; +} + diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc new file mode 100644 index 0000000000..2616706ee5 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" + +#include + +#include "absl/types/optional.h" +#include "rtc_base/checks.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/system/file_wrapper.h" + +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/debug_dump.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +#if WEBRTC_ENABLE_PROTOBUF +namespace { + +using audio_network_adaptor::debug_dump::EncoderRuntimeConfig; +using audio_network_adaptor::debug_dump::Event; +using audio_network_adaptor::debug_dump::NetworkMetrics; + +void DumpEventToFile(const Event& event, FileWrapper* dump_file) { + RTC_CHECK(dump_file->is_open()); + std::string dump_data; + event.SerializeToString(&dump_data); + int32_t size = rtc::checked_cast(event.ByteSizeLong()); + dump_file->Write(&size, sizeof(size)); + dump_file->Write(dump_data.data(), dump_data.length()); +} + +} // namespace +#endif // WEBRTC_ENABLE_PROTOBUF + +class DebugDumpWriterImpl final : public DebugDumpWriter { + public: + explicit DebugDumpWriterImpl(FILE* file_handle); + ~DebugDumpWriterImpl() override = default; + + void DumpEncoderRuntimeConfig(const AudioEncoderRuntimeConfig& config, + int64_t timestamp) override; + + void DumpNetworkMetrics(const Controller::NetworkMetrics& metrics, + int64_t timestamp) override; + +#if WEBRTC_ENABLE_PROTOBUF + void DumpControllerManagerConfig( + const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp) override; +#endif + + private: + FileWrapper dump_file_; +}; + +DebugDumpWriterImpl::DebugDumpWriterImpl(FILE* file_handle) { +#if WEBRTC_ENABLE_PROTOBUF + dump_file_ = FileWrapper(file_handle); + RTC_CHECK(dump_file_.is_open()); +#else + RTC_DCHECK_NOTREACHED(); +#endif +} + +void DebugDumpWriterImpl::DumpNetworkMetrics( + const Controller::NetworkMetrics& metrics, + int64_t timestamp) { +#if WEBRTC_ENABLE_PROTOBUF + Event event; + event.set_timestamp(timestamp); + event.set_type(Event::NETWORK_METRICS); + auto dump_metrics = event.mutable_network_metrics(); + + if (metrics.uplink_bandwidth_bps) + dump_metrics->set_uplink_bandwidth_bps(*metrics.uplink_bandwidth_bps); + + if (metrics.uplink_packet_loss_fraction) { + dump_metrics->set_uplink_packet_loss_fraction( + *metrics.uplink_packet_loss_fraction); + } + + if (metrics.target_audio_bitrate_bps) { + dump_metrics->set_target_audio_bitrate_bps( + *metrics.target_audio_bitrate_bps); + } + + if (metrics.rtt_ms) + dump_metrics->set_rtt_ms(*metrics.rtt_ms); + + DumpEventToFile(event, &dump_file_); +#endif // WEBRTC_ENABLE_PROTOBUF +} + +void DebugDumpWriterImpl::DumpEncoderRuntimeConfig( + const AudioEncoderRuntimeConfig& config, + int64_t timestamp) { +#if WEBRTC_ENABLE_PROTOBUF + Event event; + event.set_timestamp(timestamp); + event.set_type(Event::ENCODER_RUNTIME_CONFIG); + auto dump_config = event.mutable_encoder_runtime_config(); + + if (config.bitrate_bps) + dump_config->set_bitrate_bps(*config.bitrate_bps); + + if (config.frame_length_ms) + dump_config->set_frame_length_ms(*config.frame_length_ms); + + if (config.uplink_packet_loss_fraction) { + dump_config->set_uplink_packet_loss_fraction( + *config.uplink_packet_loss_fraction); + } + + if (config.enable_fec) + dump_config->set_enable_fec(*config.enable_fec); + + if (config.enable_dtx) + dump_config->set_enable_dtx(*config.enable_dtx); + + if (config.num_channels) + dump_config->set_num_channels(*config.num_channels); + + DumpEventToFile(event, &dump_file_); +#endif // WEBRTC_ENABLE_PROTOBUF +} + +#if WEBRTC_ENABLE_PROTOBUF +void DebugDumpWriterImpl::DumpControllerManagerConfig( + const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp) { + Event event; + event.set_timestamp(timestamp); + event.set_type(Event::CONTROLLER_MANAGER_CONFIG); + event.mutable_controller_manager_config()->CopyFrom( + controller_manager_config); + DumpEventToFile(event, &dump_file_); +} +#endif // WEBRTC_ENABLE_PROTOBUF + +std::unique_ptr DebugDumpWriter::Create(FILE* file_handle) { + return std::unique_ptr(new DebugDumpWriterImpl(file_handle)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h new file mode 100644 index 0000000000..8fdf2f7728 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DEBUG_DUMP_WRITER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DEBUG_DUMP_WRITER_H_ + +#include + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/system/file_wrapper.h" +#if WEBRTC_ENABLE_PROTOBUF +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h" +#else +#include "modules/audio_coding/audio_network_adaptor/config.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +class DebugDumpWriter { + public: + static std::unique_ptr Create(FILE* file_handle); + + virtual ~DebugDumpWriter() = default; + + virtual void DumpEncoderRuntimeConfig(const AudioEncoderRuntimeConfig& config, + int64_t timestamp) = 0; + + virtual void DumpNetworkMetrics(const Controller::NetworkMetrics& metrics, + int64_t timestamp) = 0; + +#if WEBRTC_ENABLE_PROTOBUF + virtual void DumpControllerManagerConfig( + const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp) = 0; +#endif +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DEBUG_DUMP_WRITER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc new file mode 100644 index 0000000000..b0a7d5d59d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/dtx_controller.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +DtxController::Config::Config(bool initial_dtx_enabled, + int dtx_enabling_bandwidth_bps, + int dtx_disabling_bandwidth_bps) + : initial_dtx_enabled(initial_dtx_enabled), + dtx_enabling_bandwidth_bps(dtx_enabling_bandwidth_bps), + dtx_disabling_bandwidth_bps(dtx_disabling_bandwidth_bps) {} + +DtxController::DtxController(const Config& config) + : config_(config), dtx_enabled_(config_.initial_dtx_enabled) {} + +DtxController::~DtxController() = default; + +void DtxController::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; +} + +void DtxController::MakeDecision(AudioEncoderRuntimeConfig* config) { + // Decision on `enable_dtx` should not have been made. + RTC_DCHECK(!config->enable_dtx); + + if (uplink_bandwidth_bps_) { + if (dtx_enabled_ && + *uplink_bandwidth_bps_ >= config_.dtx_disabling_bandwidth_bps) { + dtx_enabled_ = false; + } else if (!dtx_enabled_ && + *uplink_bandwidth_bps_ <= config_.dtx_enabling_bandwidth_bps) { + dtx_enabled_ = true; + } + } + config->enable_dtx = dtx_enabled_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.h new file mode 100644 index 0000000000..b8a8e476e4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DTX_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DTX_CONTROLLER_H_ + +#include "absl/types/optional.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { + +class DtxController final : public Controller { + public: + struct Config { + Config(bool initial_dtx_enabled, + int dtx_enabling_bandwidth_bps, + int dtx_disabling_bandwidth_bps); + bool initial_dtx_enabled; + // Uplink bandwidth below which DTX should be switched on. + int dtx_enabling_bandwidth_bps; + // Uplink bandwidth above which DTX should be switched off. + int dtx_disabling_bandwidth_bps; + }; + + explicit DtxController(const Config& config); + + ~DtxController() override; + + DtxController(const DtxController&) = delete; + DtxController& operator=(const DtxController&) = delete; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + const Config config_; + bool dtx_enabled_; + absl::optional uplink_bandwidth_bps_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_DTX_CONTROLLER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller_unittest.cc new file mode 100644 index 0000000000..567df6f76e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller_unittest.cc @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/dtx_controller.h" + +#include + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kDtxEnablingBandwidthBps = 55000; +constexpr int kDtxDisablingBandwidthBps = 65000; +constexpr int kMediumBandwidthBps = + (kDtxEnablingBandwidthBps + kDtxDisablingBandwidthBps) / 2; + +std::unique_ptr CreateController(int initial_dtx_enabled) { + std::unique_ptr controller(new DtxController( + DtxController::Config(initial_dtx_enabled, kDtxEnablingBandwidthBps, + kDtxDisablingBandwidthBps))); + return controller; +} + +void CheckDecision(DtxController* controller, + const absl::optional& uplink_bandwidth_bps, + bool expected_dtx_enabled) { + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + AudioEncoderRuntimeConfig config; + controller->MakeDecision(&config); + EXPECT_EQ(expected_dtx_enabled, config.enable_dtx); +} + +} // namespace + +TEST(DtxControllerTest, OutputInitValueWhenUplinkBandwidthUnknown) { + constexpr bool kInitialDtxEnabled = true; + auto controller = CreateController(kInitialDtxEnabled); + CheckDecision(controller.get(), absl::nullopt, kInitialDtxEnabled); +} + +TEST(DtxControllerTest, TurnOnDtxForLowUplinkBandwidth) { + auto controller = CreateController(false); + CheckDecision(controller.get(), kDtxEnablingBandwidthBps, true); +} + +TEST(DtxControllerTest, TurnOffDtxForHighUplinkBandwidth) { + auto controller = CreateController(true); + CheckDecision(controller.get(), kDtxDisablingBandwidthBps, false); +} + +TEST(DtxControllerTest, MaintainDtxOffForMediumUplinkBandwidth) { + auto controller = CreateController(false); + CheckDecision(controller.get(), kMediumBandwidthBps, false); +} + +TEST(DtxControllerTest, MaintainDtxOnForMediumUplinkBandwidth) { + auto controller = CreateController(true); + CheckDecision(controller.get(), kMediumBandwidthBps, true); +} + +TEST(DtxControllerTest, CheckBehaviorOnChangingUplinkBandwidth) { + auto controller = CreateController(false); + CheckDecision(controller.get(), kMediumBandwidthBps, false); + CheckDecision(controller.get(), kDtxEnablingBandwidthBps, true); + CheckDecision(controller.get(), kMediumBandwidthBps, true); + CheckDecision(controller.get(), kDtxDisablingBandwidthBps, false); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc new file mode 100644 index 0000000000..0a79484a16 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/event_log_writer.h" + +#include + +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/rtc_event_log/rtc_event.h" +#include "api/rtc_event_log/rtc_event_log.h" +#include "logging/rtc_event_log/events/rtc_event_audio_network_adaptation.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +EventLogWriter::EventLogWriter(RtcEventLog* event_log, + int min_bitrate_change_bps, + float min_bitrate_change_fraction, + float min_packet_loss_change_fraction) + : event_log_(event_log), + min_bitrate_change_bps_(min_bitrate_change_bps), + min_bitrate_change_fraction_(min_bitrate_change_fraction), + min_packet_loss_change_fraction_(min_packet_loss_change_fraction) { + RTC_DCHECK(event_log_); +} + +EventLogWriter::~EventLogWriter() = default; + +void EventLogWriter::MaybeLogEncoderConfig( + const AudioEncoderRuntimeConfig& config) { + if (last_logged_config_.num_channels != config.num_channels) + return LogEncoderConfig(config); + if (last_logged_config_.enable_dtx != config.enable_dtx) + return LogEncoderConfig(config); + if (last_logged_config_.enable_fec != config.enable_fec) + return LogEncoderConfig(config); + if (last_logged_config_.frame_length_ms != config.frame_length_ms) + return LogEncoderConfig(config); + if ((!last_logged_config_.bitrate_bps && config.bitrate_bps) || + (last_logged_config_.bitrate_bps && config.bitrate_bps && + std::abs(*last_logged_config_.bitrate_bps - *config.bitrate_bps) >= + std::min(static_cast(*last_logged_config_.bitrate_bps * + min_bitrate_change_fraction_), + min_bitrate_change_bps_))) { + return LogEncoderConfig(config); + } + if ((!last_logged_config_.uplink_packet_loss_fraction && + config.uplink_packet_loss_fraction) || + (last_logged_config_.uplink_packet_loss_fraction && + config.uplink_packet_loss_fraction && + fabs(*last_logged_config_.uplink_packet_loss_fraction - + *config.uplink_packet_loss_fraction) >= + min_packet_loss_change_fraction_ * + *last_logged_config_.uplink_packet_loss_fraction)) { + return LogEncoderConfig(config); + } +} + +void EventLogWriter::LogEncoderConfig(const AudioEncoderRuntimeConfig& config) { + auto config_copy = std::make_unique(config); + event_log_->Log( + std::make_unique(std::move(config_copy))); + last_logged_config_ = config; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.h new file mode 100644 index 0000000000..a147311fc7 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_EVENT_LOG_WRITER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_EVENT_LOG_WRITER_H_ + +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { +class RtcEventLog; + +class EventLogWriter final { + public: + EventLogWriter(RtcEventLog* event_log, + int min_bitrate_change_bps, + float min_bitrate_change_fraction, + float min_packet_loss_change_fraction); + ~EventLogWriter(); + + EventLogWriter(const EventLogWriter&) = delete; + EventLogWriter& operator=(const EventLogWriter&) = delete; + + void MaybeLogEncoderConfig(const AudioEncoderRuntimeConfig& config); + + private: + void LogEncoderConfig(const AudioEncoderRuntimeConfig& config); + + RtcEventLog* const event_log_; + const int min_bitrate_change_bps_; + const float min_bitrate_change_fraction_; + const float min_packet_loss_change_fraction_; + AudioEncoderRuntimeConfig last_logged_config_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_EVENT_LOG_WRITER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer_unittest.cc new file mode 100644 index 0000000000..2c344534ea --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer_unittest.cc @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/event_log_writer.h" + +#include + +#include "logging/rtc_event_log/events/rtc_event_audio_network_adaptation.h" +#include "logging/rtc_event_log/mock/mock_rtc_event_log.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kMinBitrateChangeBps = 5000; +constexpr float kMinPacketLossChangeFraction = 0.5; +constexpr float kMinBitrateChangeFraction = 0.25; + +constexpr int kHighBitrateBps = 70000; +constexpr int kLowBitrateBps = 10000; +constexpr int kFrameLengthMs = 60; +constexpr bool kEnableFec = true; +constexpr bool kEnableDtx = true; +constexpr float kPacketLossFraction = 0.05f; +constexpr size_t kNumChannels = 1; + +MATCHER_P(IsRtcEventAnaConfigEqualTo, config, "") { + if (arg->GetType() != RtcEvent::Type::AudioNetworkAdaptation) { + return false; + } + auto ana_event = static_cast(arg); + return ana_event->config() == config; +} + +struct EventLogWriterStates { + std::unique_ptr event_log_writer; + std::unique_ptr> event_log; + AudioEncoderRuntimeConfig runtime_config; +}; + +EventLogWriterStates CreateEventLogWriter() { + EventLogWriterStates state; + state.event_log.reset(new ::testing::StrictMock()); + state.event_log_writer.reset(new EventLogWriter( + state.event_log.get(), kMinBitrateChangeBps, kMinBitrateChangeFraction, + kMinPacketLossChangeFraction)); + state.runtime_config.bitrate_bps = kHighBitrateBps; + state.runtime_config.frame_length_ms = kFrameLengthMs; + state.runtime_config.uplink_packet_loss_fraction = kPacketLossFraction; + state.runtime_config.enable_fec = kEnableFec; + state.runtime_config.enable_dtx = kEnableDtx; + state.runtime_config.num_channels = kNumChannels; + return state; +} +} // namespace + +TEST(EventLogWriterTest, FirstConfigIsLogged) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, SameConfigIsNotLogged) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogFecStateChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + + state.runtime_config.enable_fec = !kEnableFec; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogDtxStateChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + + state.runtime_config.enable_dtx = !kEnableDtx; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogChannelChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + + state.runtime_config.num_channels = kNumChannels + 1; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogFrameLengthChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + + state.runtime_config.frame_length_ms = 20; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, DoNotLogSmallBitrateChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.bitrate_bps = kHighBitrateBps + kMinBitrateChangeBps - 1; + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogLargeBitrateChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + // At high bitrate, the min fraction rule requires a larger change than the + // min change rule. We make sure that the min change rule applies. + RTC_DCHECK_GT(kHighBitrateBps * kMinBitrateChangeFraction, + kMinBitrateChangeBps); + state.runtime_config.bitrate_bps = kHighBitrateBps + kMinBitrateChangeBps; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogMinBitrateChangeFractionOnLowBitrateChange) { + auto state = CreateEventLogWriter(); + state.runtime_config.bitrate_bps = kLowBitrateBps; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + // At high bitrate, the min change rule requires a larger change than the min + // fraction rule. We make sure that the min fraction rule applies. + state.runtime_config.bitrate_bps = + kLowBitrateBps + kLowBitrateBps * kMinBitrateChangeFraction; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, DoNotLogSmallPacketLossFractionChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.uplink_packet_loss_fraction = + kPacketLossFraction + kMinPacketLossChangeFraction * kPacketLossFraction - + 0.001f; + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogLargePacketLossFractionChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.uplink_packet_loss_fraction = + kPacketLossFraction + kMinPacketLossChangeFraction * kPacketLossFraction; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogJustOnceOnMultipleChanges) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.uplink_packet_loss_fraction = + kPacketLossFraction + kMinPacketLossChangeFraction * kPacketLossFraction; + state.runtime_config.frame_length_ms = 20; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); +} + +TEST(EventLogWriterTest, LogAfterGradualChange) { + auto state = CreateEventLogWriter(); + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + state.runtime_config.bitrate_bps = kHighBitrateBps + kMinBitrateChangeBps; + EXPECT_CALL(*state.event_log, + LogProxy(IsRtcEventAnaConfigEqualTo(state.runtime_config))) + .Times(1); + for (int bitrate_bps = kHighBitrateBps; + bitrate_bps <= kHighBitrateBps + kMinBitrateChangeBps; bitrate_bps++) { + state.runtime_config.bitrate_bps = bitrate_bps; + state.event_log_writer->MaybeLogEncoderConfig(state.runtime_config); + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc new file mode 100644 index 0000000000..c5e5fa76e3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { +class NullSmoothingFilter final : public SmoothingFilter { + public: + void AddSample(float sample) override { last_sample_ = sample; } + + absl::optional GetAverage() override { return last_sample_; } + + bool SetTimeConstantMs(int time_constant_ms) override { + RTC_DCHECK_NOTREACHED(); + return false; + } + + private: + absl::optional last_sample_; +}; +} // namespace + +FecControllerPlrBased::Config::Config( + bool initial_fec_enabled, + const ThresholdCurve& fec_enabling_threshold, + const ThresholdCurve& fec_disabling_threshold, + int time_constant_ms) + : initial_fec_enabled(initial_fec_enabled), + fec_enabling_threshold(fec_enabling_threshold), + fec_disabling_threshold(fec_disabling_threshold), + time_constant_ms(time_constant_ms) {} + +FecControllerPlrBased::FecControllerPlrBased( + const Config& config, + std::unique_ptr smoothing_filter) + : config_(config), + fec_enabled_(config.initial_fec_enabled), + packet_loss_smoother_(std::move(smoothing_filter)) { + RTC_DCHECK(config_.fec_disabling_threshold <= config_.fec_enabling_threshold); +} + +FecControllerPlrBased::FecControllerPlrBased(const Config& config) + : FecControllerPlrBased( + config, + webrtc::field_trial::FindFullName("UseTwccPlrForAna") == "Enabled" + ? std::unique_ptr(new NullSmoothingFilter()) + : std::unique_ptr( + new SmoothingFilterImpl(config.time_constant_ms))) {} + +FecControllerPlrBased::~FecControllerPlrBased() = default; + +void FecControllerPlrBased::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; + if (network_metrics.uplink_packet_loss_fraction) { + packet_loss_smoother_->AddSample( + *network_metrics.uplink_packet_loss_fraction); + } +} + +void FecControllerPlrBased::MakeDecision(AudioEncoderRuntimeConfig* config) { + RTC_DCHECK(!config->enable_fec); + RTC_DCHECK(!config->uplink_packet_loss_fraction); + + const auto& packet_loss = packet_loss_smoother_->GetAverage(); + + fec_enabled_ = fec_enabled_ ? !FecDisablingDecision(packet_loss) + : FecEnablingDecision(packet_loss); + + config->enable_fec = fec_enabled_; + + config->uplink_packet_loss_fraction = packet_loss ? *packet_loss : 0.0; +} + +bool FecControllerPlrBased::FecEnablingDecision( + const absl::optional& packet_loss) const { + if (!uplink_bandwidth_bps_ || !packet_loss) { + return false; + } else { + // Enable when above the curve or exactly on it. + return !config_.fec_enabling_threshold.IsBelowCurve( + {static_cast(*uplink_bandwidth_bps_), *packet_loss}); + } +} + +bool FecControllerPlrBased::FecDisablingDecision( + const absl::optional& packet_loss) const { + if (!uplink_bandwidth_bps_ || !packet_loss) { + return false; + } else { + // Disable when below the curve. + return config_.fec_disabling_threshold.IsBelowCurve( + {static_cast(*uplink_bandwidth_bps_), *packet_loss}); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h new file mode 100644 index 0000000000..0c57ad1d1e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_PLR_BASED_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_PLR_BASED_H_ + +#include + +#include "absl/types/optional.h" +#include "common_audio/smoothing_filter.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" +#include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" + +namespace webrtc { + +class FecControllerPlrBased final : public Controller { + public: + struct Config { + // `fec_enabling_threshold` defines a curve, above which FEC should be + // enabled. `fec_disabling_threshold` defines a curve, under which FEC + // should be disabled. See below + // + // packet-loss ^ | | + // | | | FEC + // | \ \ ON + // | FEC \ \_______ fec_enabling_threshold + // | OFF \_________ fec_disabling_threshold + // |-----------------> bandwidth + Config(bool initial_fec_enabled, + const ThresholdCurve& fec_enabling_threshold, + const ThresholdCurve& fec_disabling_threshold, + int time_constant_ms); + bool initial_fec_enabled; + ThresholdCurve fec_enabling_threshold; + ThresholdCurve fec_disabling_threshold; + int time_constant_ms; + }; + + // Dependency injection for testing. + FecControllerPlrBased(const Config& config, + std::unique_ptr smoothing_filter); + + explicit FecControllerPlrBased(const Config& config); + + ~FecControllerPlrBased() override; + + FecControllerPlrBased(const FecControllerPlrBased&) = delete; + FecControllerPlrBased& operator=(const FecControllerPlrBased&) = delete; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + bool FecEnablingDecision(const absl::optional& packet_loss) const; + bool FecDisablingDecision(const absl::optional& packet_loss) const; + + const Config config_; + bool fec_enabled_; + absl::optional uplink_bandwidth_bps_; + const std::unique_ptr packet_loss_smoother_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FEC_CONTROLLER_PLR_BASED_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based_unittest.cc new file mode 100644 index 0000000000..743b087163 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based_unittest.cc @@ -0,0 +1,489 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h" + +#include + +#include "common_audio/mocks/mock_smoothing_filter.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; + +namespace { + +// The test uses the following settings: +// +// packet-loss ^ | | +// | A| C| FEC +// | \ \ ON +// | FEC \ D\_______ +// | OFF B\_________ +// |-----------------> bandwidth +// +// A : (kDisablingBandwidthLow, kDisablingPacketLossAtLowBw) +// B : (kDisablingBandwidthHigh, kDisablingPacketLossAtHighBw) +// C : (kEnablingBandwidthLow, kEnablingPacketLossAtLowBw) +// D : (kEnablingBandwidthHigh, kEnablingPacketLossAtHighBw) + +constexpr int kDisablingBandwidthLow = 15000; +constexpr float kDisablingPacketLossAtLowBw = 0.08f; +constexpr int kDisablingBandwidthHigh = 64000; +constexpr float kDisablingPacketLossAtHighBw = 0.01f; +constexpr int kEnablingBandwidthLow = 17000; +constexpr float kEnablingPacketLossAtLowBw = 0.1f; +constexpr int kEnablingBandwidthHigh = 64000; +constexpr float kEnablingPacketLossAtHighBw = 0.05f; + +constexpr float kEpsilon = 1e-5f; + +struct FecControllerPlrBasedTestStates { + std::unique_ptr controller; + MockSmoothingFilter* packet_loss_smoother; +}; + +FecControllerPlrBasedTestStates CreateFecControllerPlrBased( + bool initial_fec_enabled, + const ThresholdCurve& enabling_curve, + const ThresholdCurve& disabling_curve) { + FecControllerPlrBasedTestStates states; + std::unique_ptr mock_smoothing_filter( + new NiceMock()); + states.packet_loss_smoother = mock_smoothing_filter.get(); + states.controller.reset(new FecControllerPlrBased( + FecControllerPlrBased::Config(initial_fec_enabled, enabling_curve, + disabling_curve, 0), + std::move(mock_smoothing_filter))); + return states; +} + +FecControllerPlrBasedTestStates CreateFecControllerPlrBased( + bool initial_fec_enabled) { + return CreateFecControllerPlrBased( + initial_fec_enabled, + ThresholdCurve(kEnablingBandwidthLow, kEnablingPacketLossAtLowBw, + kEnablingBandwidthHigh, kEnablingPacketLossAtHighBw), + ThresholdCurve(kDisablingBandwidthLow, kDisablingPacketLossAtLowBw, + kDisablingBandwidthHigh, kDisablingPacketLossAtHighBw)); +} + +void UpdateNetworkMetrics(FecControllerPlrBasedTestStates* states, + const absl::optional& uplink_bandwidth_bps, + const absl::optional& uplink_packet_loss) { + // UpdateNetworkMetrics can accept multiple network metric updates at once. + // However, currently, the most used case is to update one metric at a time. + // To reflect this fact, we separate the calls. + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + states->controller->UpdateNetworkMetrics(network_metrics); + } + if (uplink_packet_loss) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_packet_loss_fraction = uplink_packet_loss; + EXPECT_CALL(*states->packet_loss_smoother, AddSample(*uplink_packet_loss)); + states->controller->UpdateNetworkMetrics(network_metrics); + // This is called during CheckDecision(). + EXPECT_CALL(*states->packet_loss_smoother, GetAverage()) + .WillOnce(Return(*uplink_packet_loss)); + } +} + +// Checks that the FEC decision and `uplink_packet_loss_fraction` given by +// `states->controller->MakeDecision` matches `expected_enable_fec` and +// `expected_uplink_packet_loss_fraction`, respectively. +void CheckDecision(FecControllerPlrBasedTestStates* states, + bool expected_enable_fec, + float expected_uplink_packet_loss_fraction) { + AudioEncoderRuntimeConfig config; + states->controller->MakeDecision(&config); + EXPECT_EQ(expected_enable_fec, config.enable_fec); + EXPECT_EQ(expected_uplink_packet_loss_fraction, + config.uplink_packet_loss_fraction); +} + +} // namespace + +TEST(FecControllerPlrBasedTest, OutputInitValueBeforeAnyInputsAreReceived) { + for (bool initial_fec_enabled : {false, true}) { + auto states = CreateFecControllerPlrBased(initial_fec_enabled); + CheckDecision(&states, initial_fec_enabled, 0); + } +} + +TEST(FecControllerPlrBasedTest, OutputInitValueWhenUplinkBandwidthUnknown) { + // Regardless of the initial FEC state and the packet-loss rate, + // the initial FEC state is maintained as long as the BWE is unknown. + for (bool initial_fec_enabled : {false, true}) { + for (float packet_loss : + {kDisablingPacketLossAtLowBw - kEpsilon, kDisablingPacketLossAtLowBw, + kDisablingPacketLossAtLowBw + kEpsilon, + kEnablingPacketLossAtLowBw - kEpsilon, kEnablingPacketLossAtLowBw, + kEnablingPacketLossAtLowBw + kEpsilon}) { + auto states = CreateFecControllerPlrBased(initial_fec_enabled); + UpdateNetworkMetrics(&states, absl::nullopt, packet_loss); + CheckDecision(&states, initial_fec_enabled, packet_loss); + } + } +} + +TEST(FecControllerPlrBasedTest, + OutputInitValueWhenUplinkPacketLossFractionUnknown) { + // Regardless of the initial FEC state and the BWE, the initial FEC state + // is maintained as long as the packet-loss rate is unknown. + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {kDisablingBandwidthLow - 1, kDisablingBandwidthLow, + kDisablingBandwidthLow + 1, kEnablingBandwidthLow - 1, + kEnablingBandwidthLow, kEnablingBandwidthLow + 1}) { + auto states = CreateFecControllerPlrBased(initial_fec_enabled); + UpdateNetworkMetrics(&states, bandwidth, absl::nullopt); + CheckDecision(&states, initial_fec_enabled, 0.0); + } + } +} + +TEST(FecControllerPlrBasedTest, EnableFecForHighBandwidth) { + auto states = CreateFecControllerPlrBased(false); + UpdateNetworkMetrics(&states, kEnablingBandwidthHigh, + kEnablingPacketLossAtHighBw); + CheckDecision(&states, true, kEnablingPacketLossAtHighBw); +} + +TEST(FecControllerPlrBasedTest, UpdateMultipleNetworkMetricsAtOnce) { + // This test is similar to EnableFecForHighBandwidth. But instead of + // using ::UpdateNetworkMetrics(...), which calls + // FecControllerPlrBased::UpdateNetworkMetrics(...) multiple times, we + // we call it only once. This is to verify that + // FecControllerPlrBased::UpdateNetworkMetrics(...) can handle multiple + // network updates at once. This is, however, not a common use case in current + // audio_network_adaptor_impl.cc. + auto states = CreateFecControllerPlrBased(false); + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = kEnablingBandwidthHigh; + network_metrics.uplink_packet_loss_fraction = kEnablingPacketLossAtHighBw; + EXPECT_CALL(*states.packet_loss_smoother, GetAverage()) + .WillOnce(Return(kEnablingPacketLossAtHighBw)); + states.controller->UpdateNetworkMetrics(network_metrics); + CheckDecision(&states, true, kEnablingPacketLossAtHighBw); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOffForHighBandwidth) { + auto states = CreateFecControllerPlrBased(false); + constexpr float kPacketLoss = kEnablingPacketLossAtHighBw * 0.99f; + UpdateNetworkMetrics(&states, kEnablingBandwidthHigh, kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, EnableFecForMediumBandwidth) { + auto states = CreateFecControllerPlrBased(false); + constexpr float kPacketLoss = + (kEnablingPacketLossAtLowBw + kEnablingPacketLossAtHighBw) / 2.0; + UpdateNetworkMetrics(&states, + (kEnablingBandwidthHigh + kEnablingBandwidthLow) / 2, + kPacketLoss); + CheckDecision(&states, true, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOffForMediumBandwidth) { + auto states = CreateFecControllerPlrBased(false); + constexpr float kPacketLoss = + kEnablingPacketLossAtLowBw * 0.49f + kEnablingPacketLossAtHighBw * 0.51f; + UpdateNetworkMetrics(&states, + (kEnablingBandwidthHigh + kEnablingBandwidthLow) / 2, + kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, EnableFecForLowBandwidth) { + auto states = CreateFecControllerPlrBased(false); + UpdateNetworkMetrics(&states, kEnablingBandwidthLow, + kEnablingPacketLossAtLowBw); + CheckDecision(&states, true, kEnablingPacketLossAtLowBw); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOffForLowBandwidth) { + auto states = CreateFecControllerPlrBased(false); + constexpr float kPacketLoss = kEnablingPacketLossAtLowBw * 0.99f; + UpdateNetworkMetrics(&states, kEnablingBandwidthLow, kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOffForVeryLowBandwidth) { + auto states = CreateFecControllerPlrBased(false); + // Below `kEnablingBandwidthLow`, no packet loss fraction can cause FEC to + // turn on. + UpdateNetworkMetrics(&states, kEnablingBandwidthLow - 1, 1.0); + CheckDecision(&states, false, 1.0); +} + +TEST(FecControllerPlrBasedTest, DisableFecForHighBandwidth) { + auto states = CreateFecControllerPlrBased(true); + constexpr float kPacketLoss = kDisablingPacketLossAtHighBw - kEpsilon; + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh, kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOnForHighBandwidth) { + // Note: Disabling happens when the value is strictly below the threshold. + auto states = CreateFecControllerPlrBased(true); + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh, + kDisablingPacketLossAtHighBw); + CheckDecision(&states, true, kDisablingPacketLossAtHighBw); +} + +TEST(FecControllerPlrBasedTest, DisableFecOnMediumBandwidth) { + auto states = CreateFecControllerPlrBased(true); + constexpr float kPacketLoss = + (kDisablingPacketLossAtLowBw + kDisablingPacketLossAtHighBw) / 2.0f - + kEpsilon; + UpdateNetworkMetrics(&states, + (kDisablingBandwidthHigh + kDisablingBandwidthLow) / 2, + kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, MaintainFecOnForMediumBandwidth) { + auto states = CreateFecControllerPlrBased(true); + constexpr float kPacketLoss = kDisablingPacketLossAtLowBw * 0.51f + + kDisablingPacketLossAtHighBw * 0.49f - kEpsilon; + UpdateNetworkMetrics(&states, + (kEnablingBandwidthHigh + kDisablingBandwidthLow) / 2, + kPacketLoss); + CheckDecision(&states, true, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, DisableFecForLowBandwidth) { + auto states = CreateFecControllerPlrBased(true); + constexpr float kPacketLoss = kDisablingPacketLossAtLowBw - kEpsilon; + UpdateNetworkMetrics(&states, kDisablingBandwidthLow, kPacketLoss); + CheckDecision(&states, false, kPacketLoss); +} + +TEST(FecControllerPlrBasedTest, DisableFecForVeryLowBandwidth) { + auto states = CreateFecControllerPlrBased(true); + // Below `kEnablingBandwidthLow`, any packet loss fraction can cause FEC to + // turn off. + UpdateNetworkMetrics(&states, kDisablingBandwidthLow - 1, 1.0); + CheckDecision(&states, false, 1.0); +} + +TEST(FecControllerPlrBasedTest, CheckBehaviorOnChangingNetworkMetrics) { + // In this test, we let the network metrics to traverse from 1 to 5. + // packet-loss ^ 1 | | + // | | 2| + // | \ \ 3 + // | \4 \_______ + // | \_________ + // |---------5-------> bandwidth + + auto states = CreateFecControllerPlrBased(true); + UpdateNetworkMetrics(&states, kDisablingBandwidthLow - 1, 1.0); + CheckDecision(&states, false, 1.0); + + UpdateNetworkMetrics(&states, kEnablingBandwidthLow, + kEnablingPacketLossAtLowBw * 0.99f); + CheckDecision(&states, false, kEnablingPacketLossAtLowBw * 0.99f); + + UpdateNetworkMetrics(&states, kEnablingBandwidthHigh, + kEnablingPacketLossAtHighBw); + CheckDecision(&states, true, kEnablingPacketLossAtHighBw); + + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh, + kDisablingPacketLossAtHighBw); + CheckDecision(&states, true, kDisablingPacketLossAtHighBw); + + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh + 1, 0.0); + CheckDecision(&states, false, 0.0); +} + +TEST(FecControllerPlrBasedTest, CheckBehaviorOnSpecialCurves) { + // We test a special configuration, where the points to define the FEC + // enabling/disabling curves are placed like the following, otherwise the test + // is the same as CheckBehaviorOnChangingNetworkMetrics. + // + // packet-loss ^ | | + // | | C| + // | | | + // | | D|_______ + // | A|___B______ + // |-----------------> bandwidth + + constexpr int kEnablingBandwidthHigh = kEnablingBandwidthLow; + constexpr float kDisablingPacketLossAtLowBw = kDisablingPacketLossAtHighBw; + FecControllerPlrBasedTestStates states; + std::unique_ptr mock_smoothing_filter( + new NiceMock()); + states.packet_loss_smoother = mock_smoothing_filter.get(); + states.controller.reset(new FecControllerPlrBased( + FecControllerPlrBased::Config( + true, + ThresholdCurve(kEnablingBandwidthLow, kEnablingPacketLossAtLowBw, + kEnablingBandwidthHigh, kEnablingPacketLossAtHighBw), + ThresholdCurve(kDisablingBandwidthLow, kDisablingPacketLossAtLowBw, + kDisablingBandwidthHigh, kDisablingPacketLossAtHighBw), + 0), + std::move(mock_smoothing_filter))); + + UpdateNetworkMetrics(&states, kDisablingBandwidthLow - 1, 1.0); + CheckDecision(&states, false, 1.0); + + UpdateNetworkMetrics(&states, kEnablingBandwidthLow, + kEnablingPacketLossAtHighBw * 0.99f); + CheckDecision(&states, false, kEnablingPacketLossAtHighBw * 0.99f); + + UpdateNetworkMetrics(&states, kEnablingBandwidthHigh, + kEnablingPacketLossAtHighBw); + CheckDecision(&states, true, kEnablingPacketLossAtHighBw); + + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh, + kDisablingPacketLossAtHighBw); + CheckDecision(&states, true, kDisablingPacketLossAtHighBw); + + UpdateNetworkMetrics(&states, kDisablingBandwidthHigh + 1, 0.0); + CheckDecision(&states, false, 0.0); +} + +TEST(FecControllerPlrBasedTest, SingleThresholdCurveForEnablingAndDisabling) { + // Note: To avoid numerical errors, keep kPacketLossAtLowBw and + // kPacketLossAthighBw as (negative) integer powers of 2. + // This is mostly relevant for the O3 case. + constexpr int kBandwidthLow = 10000; + constexpr float kPacketLossAtLowBw = 0.25f; + constexpr int kBandwidthHigh = 20000; + constexpr float kPacketLossAtHighBw = 0.125f; + auto curve = ThresholdCurve(kBandwidthLow, kPacketLossAtLowBw, kBandwidthHigh, + kPacketLossAtHighBw); + + // B* stands for "below-curve", O* for "on-curve", and A* for "above-curve". + // + // // + // packet-loss ^ // + // | | // + // | B1 O1 // + // | | // + // | O2 // + // | \ A1 // + // | \ // + // | O3 A2 // + // | B2 \ // + // | \ // + // | O4--O5---- // + // | // + // | B3 // + // |-----------------> bandwidth // + + struct NetworkState { + int bandwidth; + float packet_loss; + }; + + std::vector below{ + {kBandwidthLow - 1, kPacketLossAtLowBw + 0.1f}, // B1 + {(kBandwidthLow + kBandwidthHigh) / 2, + (kPacketLossAtLowBw + kPacketLossAtHighBw) / 2 - kEpsilon}, // B2 + {kBandwidthHigh + 1, kPacketLossAtHighBw - kEpsilon} // B3 + }; + + std::vector on{ + {kBandwidthLow, kPacketLossAtLowBw + 0.1f}, // O1 + {kBandwidthLow, kPacketLossAtLowBw}, // O2 + {(kBandwidthLow + kBandwidthHigh) / 2, + (kPacketLossAtLowBw + kPacketLossAtHighBw) / 2}, // O3 + {kBandwidthHigh, kPacketLossAtHighBw}, // O4 + {kBandwidthHigh + 1, kPacketLossAtHighBw}, // O5 + }; + + std::vector above{ + {(kBandwidthLow + kBandwidthHigh) / 2, + (kPacketLossAtLowBw + kPacketLossAtHighBw) / 2 + kEpsilon}, // A1 + {kBandwidthHigh + 1, kPacketLossAtHighBw + kEpsilon}, // A2 + }; + + // Test that FEC is turned off whenever we're below the curve, independent + // of the starting FEC state. + for (NetworkState net_state : below) { + for (bool initial_fec_enabled : {false, true}) { + auto states = + CreateFecControllerPlrBased(initial_fec_enabled, curve, curve); + UpdateNetworkMetrics(&states, net_state.bandwidth, net_state.packet_loss); + CheckDecision(&states, false, net_state.packet_loss); + } + } + + // Test that FEC is turned on whenever we're on the curve or above it, + // independent of the starting FEC state. + for (const std::vector& states_list : {on, above}) { + for (NetworkState net_state : states_list) { + for (bool initial_fec_enabled : {false, true}) { + auto states = + CreateFecControllerPlrBased(initial_fec_enabled, curve, curve); + UpdateNetworkMetrics(&states, net_state.bandwidth, + net_state.packet_loss); + CheckDecision(&states, true, net_state.packet_loss); + } + } + } +} + +TEST(FecControllerPlrBasedTest, FecAlwaysOff) { + ThresholdCurve always_off_curve(0, 1.0f + kEpsilon, 0, 1.0f + kEpsilon); + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {0, 10000}) { + for (float packet_loss : {0.0f, 0.5f, 1.0f}) { + auto states = CreateFecControllerPlrBased( + initial_fec_enabled, always_off_curve, always_off_curve); + UpdateNetworkMetrics(&states, bandwidth, packet_loss); + CheckDecision(&states, false, packet_loss); + } + } + } +} + +TEST(FecControllerPlrBasedTest, FecAlwaysOn) { + ThresholdCurve always_on_curve(0, 0.0f, 0, 0.0f); + for (bool initial_fec_enabled : {false, true}) { + for (int bandwidth : {0, 10000}) { + for (float packet_loss : {0.0f, 0.5f, 1.0f}) { + auto states = CreateFecControllerPlrBased( + initial_fec_enabled, always_on_curve, always_on_curve); + UpdateNetworkMetrics(&states, bandwidth, packet_loss); + CheckDecision(&states, true, packet_loss); + } + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(FecControllerPlrBasedDeathTest, InvalidConfig) { + FecControllerPlrBasedTestStates states; + std::unique_ptr mock_smoothing_filter( + new NiceMock()); + states.packet_loss_smoother = mock_smoothing_filter.get(); + EXPECT_DEATH( + states.controller.reset(new FecControllerPlrBased( + FecControllerPlrBased::Config( + true, + ThresholdCurve(kDisablingBandwidthLow - 1, + kEnablingPacketLossAtLowBw, kEnablingBandwidthHigh, + kEnablingPacketLossAtHighBw), + ThresholdCurve( + kDisablingBandwidthLow, kDisablingPacketLossAtLowBw, + kDisablingBandwidthHigh, kDisablingPacketLossAtHighBw), + 0), + std::move(mock_smoothing_filter))), + "Check failed"); +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc new file mode 100644 index 0000000000..c47434f9aa --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h" + +#include +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { +constexpr int kPreventOveruseMarginBps = 5000; + +int OverheadRateBps(size_t overhead_bytes_per_packet, int frame_length_ms) { + return static_cast(overhead_bytes_per_packet * 8 * 1000 / + frame_length_ms); +} +} // namespace + +FrameLengthController::Config::Config( + const std::set& encoder_frame_lengths_ms, + int initial_frame_length_ms, + int min_encoder_bitrate_bps, + float fl_increasing_packet_loss_fraction, + float fl_decreasing_packet_loss_fraction, + int fl_increase_overhead_offset, + int fl_decrease_overhead_offset, + std::map fl_changing_bandwidths_bps) + : encoder_frame_lengths_ms(encoder_frame_lengths_ms), + initial_frame_length_ms(initial_frame_length_ms), + min_encoder_bitrate_bps(min_encoder_bitrate_bps), + fl_increasing_packet_loss_fraction(fl_increasing_packet_loss_fraction), + fl_decreasing_packet_loss_fraction(fl_decreasing_packet_loss_fraction), + fl_increase_overhead_offset(fl_increase_overhead_offset), + fl_decrease_overhead_offset(fl_decrease_overhead_offset), + fl_changing_bandwidths_bps(std::move(fl_changing_bandwidths_bps)) {} + +FrameLengthController::Config::Config(const Config& other) = default; + +FrameLengthController::Config::~Config() = default; + +FrameLengthController::FrameLengthController(const Config& config) + : config_(config) { + frame_length_ms_ = std::find(config_.encoder_frame_lengths_ms.begin(), + config_.encoder_frame_lengths_ms.end(), + config_.initial_frame_length_ms); + // `encoder_frame_lengths_ms` must contain `initial_frame_length_ms`. + RTC_DCHECK(frame_length_ms_ != config_.encoder_frame_lengths_ms.end()); +} + +FrameLengthController::~FrameLengthController() = default; + +void FrameLengthController::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.uplink_bandwidth_bps) + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; + if (network_metrics.uplink_packet_loss_fraction) + uplink_packet_loss_fraction_ = network_metrics.uplink_packet_loss_fraction; + if (network_metrics.overhead_bytes_per_packet) + overhead_bytes_per_packet_ = network_metrics.overhead_bytes_per_packet; +} + +void FrameLengthController::MakeDecision(AudioEncoderRuntimeConfig* config) { + // Decision on `frame_length_ms` should not have been made. + RTC_DCHECK(!config->frame_length_ms); + + if (FrameLengthIncreasingDecision(*config)) { + prev_decision_increase_ = true; + } else if (FrameLengthDecreasingDecision(*config)) { + prev_decision_increase_ = false; + } + config->last_fl_change_increase = prev_decision_increase_; + config->frame_length_ms = *frame_length_ms_; +} + +FrameLengthController::Config::FrameLengthChange::FrameLengthChange( + int from_frame_length_ms, + int to_frame_length_ms) + : from_frame_length_ms(from_frame_length_ms), + to_frame_length_ms(to_frame_length_ms) {} + +bool FrameLengthController::Config::FrameLengthChange::operator<( + const FrameLengthChange& rhs) const { + return from_frame_length_ms < rhs.from_frame_length_ms || + (from_frame_length_ms == rhs.from_frame_length_ms && + to_frame_length_ms < rhs.to_frame_length_ms); +} + +bool FrameLengthController::FrameLengthIncreasingDecision( + const AudioEncoderRuntimeConfig& config) { + // Increase frame length if + // 1. `uplink_bandwidth_bps` is known to be smaller or equal than + // `min_encoder_bitrate_bps` plus `prevent_overuse_margin_bps` plus the + // current overhead rate OR all the following: + // 2. longer frame length is available AND + // 3. `uplink_bandwidth_bps` is known to be smaller than a threshold AND + // 4. `uplink_packet_loss_fraction` is known to be smaller than a threshold. + + // Find next frame length to which a criterion is defined to shift from + // current frame length. + auto longer_frame_length_ms = std::next(frame_length_ms_); + auto increase_threshold = config_.fl_changing_bandwidths_bps.end(); + while (longer_frame_length_ms != config_.encoder_frame_lengths_ms.end()) { + increase_threshold = config_.fl_changing_bandwidths_bps.find( + Config::FrameLengthChange(*frame_length_ms_, *longer_frame_length_ms)); + if (increase_threshold != config_.fl_changing_bandwidths_bps.end()) + break; + longer_frame_length_ms = std::next(longer_frame_length_ms); + } + + if (increase_threshold == config_.fl_changing_bandwidths_bps.end()) + return false; + + // Check that + // -(*overhead_bytes_per_packet_) <= offset <= (*overhead_bytes_per_packet_) + RTC_DCHECK( + !overhead_bytes_per_packet_ || + (overhead_bytes_per_packet_ && + static_cast(std::max(0, -config_.fl_increase_overhead_offset)) <= + *overhead_bytes_per_packet_ && + static_cast(std::max(0, config_.fl_increase_overhead_offset)) <= + *overhead_bytes_per_packet_)); + + if (uplink_bandwidth_bps_ && overhead_bytes_per_packet_ && + *uplink_bandwidth_bps_ <= + config_.min_encoder_bitrate_bps + kPreventOveruseMarginBps + + OverheadRateBps(*overhead_bytes_per_packet_ + + config_.fl_increase_overhead_offset, + *frame_length_ms_)) { + frame_length_ms_ = longer_frame_length_ms; + return true; + } + + if ((uplink_bandwidth_bps_ && + *uplink_bandwidth_bps_ <= increase_threshold->second) && + (uplink_packet_loss_fraction_ && + *uplink_packet_loss_fraction_ <= + config_.fl_increasing_packet_loss_fraction)) { + frame_length_ms_ = longer_frame_length_ms; + return true; + } + return false; +} + +bool FrameLengthController::FrameLengthDecreasingDecision( + const AudioEncoderRuntimeConfig& config) { + // Decrease frame length if + // 1. shorter frame length is available AND + // 2. `uplink_bandwidth_bps` is known to be bigger than + // `min_encoder_bitrate_bps` plus `prevent_overuse_margin_bps` plus the + // overhead which would be produced with the shorter frame length AND + // one or more of the followings: + // 3. `uplink_bandwidth_bps` is known to be larger than a threshold, + // 4. `uplink_packet_loss_fraction` is known to be larger than a threshold, + + // Find next frame length to which a criterion is defined to shift from + // current frame length. + auto shorter_frame_length_ms = frame_length_ms_; + auto decrease_threshold = config_.fl_changing_bandwidths_bps.end(); + while (shorter_frame_length_ms != config_.encoder_frame_lengths_ms.begin()) { + shorter_frame_length_ms = std::prev(shorter_frame_length_ms); + decrease_threshold = config_.fl_changing_bandwidths_bps.find( + Config::FrameLengthChange(*frame_length_ms_, *shorter_frame_length_ms)); + if (decrease_threshold != config_.fl_changing_bandwidths_bps.end()) + break; + } + + if (decrease_threshold == config_.fl_changing_bandwidths_bps.end()) + return false; + + if (uplink_bandwidth_bps_ && overhead_bytes_per_packet_ && + *uplink_bandwidth_bps_ <= + config_.min_encoder_bitrate_bps + kPreventOveruseMarginBps + + OverheadRateBps(*overhead_bytes_per_packet_ + + config_.fl_decrease_overhead_offset, + *shorter_frame_length_ms)) { + return false; + } + + if ((uplink_bandwidth_bps_ && + *uplink_bandwidth_bps_ >= decrease_threshold->second) || + (uplink_packet_loss_fraction_ && + *uplink_packet_loss_fraction_ >= + config_.fl_decreasing_packet_loss_fraction)) { + frame_length_ms_ = shorter_frame_length_ms; + return true; + } + return false; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.h new file mode 100644 index 0000000000..04693f8db7 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_H_ + +#include + +#include +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { + +// Determines target frame length based on the network metrics and the decision +// of FEC controller. +class FrameLengthController final : public Controller { + public: + struct Config { + struct FrameLengthChange { + FrameLengthChange(int from_frame_length_ms, int to_frame_length_ms); + bool operator<(const FrameLengthChange& rhs) const; + int from_frame_length_ms; + int to_frame_length_ms; + }; + Config(const std::set& encoder_frame_lengths_ms, + int initial_frame_length_ms, + int min_encoder_bitrate_bps, + float fl_increasing_packet_loss_fraction, + float fl_decreasing_packet_loss_fraction, + int fl_increase_overhead_offset, + int fl_decrease_overhead_offset, + std::map fl_changing_bandwidths_bps); + Config(const Config& other); + ~Config(); + std::set encoder_frame_lengths_ms; + int initial_frame_length_ms; + int min_encoder_bitrate_bps; + // Uplink packet loss fraction below which frame length can increase. + float fl_increasing_packet_loss_fraction; + // Uplink packet loss fraction below which frame length should decrease. + float fl_decreasing_packet_loss_fraction; + // Offset to apply to overhead calculation when increasing frame length. + int fl_increase_overhead_offset; + // Offset to apply to overhead calculation when decreasing frame length. + int fl_decrease_overhead_offset; + std::map fl_changing_bandwidths_bps; + }; + + explicit FrameLengthController(const Config& config); + + ~FrameLengthController() override; + + FrameLengthController(const FrameLengthController&) = delete; + FrameLengthController& operator=(const FrameLengthController&) = delete; + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + bool FrameLengthIncreasingDecision(const AudioEncoderRuntimeConfig& config); + + bool FrameLengthDecreasingDecision(const AudioEncoderRuntimeConfig& config); + + const Config config_; + + std::set::const_iterator frame_length_ms_; + + absl::optional uplink_bandwidth_bps_; + + absl::optional uplink_packet_loss_fraction_; + + absl::optional overhead_bytes_per_packet_; + + // True if the previous frame length decision was an increase, otherwise + // false. + bool prev_decision_increase_ = false; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_unittest.cc new file mode 100644 index 0000000000..23123934dc --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_unittest.cc @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h" + +#include +#include + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr float kFlIncreasingPacketLossFraction = 0.04f; +constexpr float kFlDecreasingPacketLossFraction = 0.05f; +constexpr int kFlIncreaseOverheadOffset = 0; +constexpr int kFlDecreaseOverheadOffset = 0; +constexpr int kMinEncoderBitrateBps = 6000; +constexpr int kPreventOveruseMarginBps = 5000; +constexpr size_t kOverheadBytesPerPacket = 20; +constexpr int kFl20msTo60msBandwidthBps = 40000; +constexpr int kFl60msTo20msBandwidthBps = 50000; +constexpr int kFl60msTo120msBandwidthBps = 30000; +constexpr int kFl120msTo60msBandwidthBps = 40000; +constexpr int kFl20msTo40msBandwidthBps = 45000; +constexpr int kFl40msTo20msBandwidthBps = 50000; +constexpr int kFl40msTo60msBandwidthBps = 40000; +constexpr int kFl60msTo40msBandwidthBps = 45000; + +constexpr int kMediumBandwidthBps = + (kFl40msTo20msBandwidthBps + kFl20msTo40msBandwidthBps) / 2; +constexpr float kMediumPacketLossFraction = + (kFlDecreasingPacketLossFraction + kFlIncreasingPacketLossFraction) / 2; +const std::set kDefaultEncoderFrameLengthsMs = {20, 40, 60, 120}; + +int VeryLowBitrate(int frame_length_ms) { + return kMinEncoderBitrateBps + kPreventOveruseMarginBps + + (kOverheadBytesPerPacket * 8 * 1000 / frame_length_ms); +} + +std::unique_ptr CreateController( + const std::map& + frame_length_change_criteria, + const std::set& encoder_frame_lengths_ms, + int initial_frame_length_ms) { + std::unique_ptr controller( + new FrameLengthController(FrameLengthController::Config( + encoder_frame_lengths_ms, initial_frame_length_ms, + kMinEncoderBitrateBps, kFlIncreasingPacketLossFraction, + kFlDecreasingPacketLossFraction, kFlIncreaseOverheadOffset, + kFlDecreaseOverheadOffset, frame_length_change_criteria))); + + return controller; +} + +std::map +CreateChangeCriteriaFor20msAnd60ms() { + return std::map{ + {FrameLengthController::Config::FrameLengthChange(20, 60), + kFl20msTo60msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 20), + kFl60msTo20msBandwidthBps}}; +} + +std::map +CreateChangeCriteriaFor20msAnd40ms() { + return std::map{ + {FrameLengthController::Config::FrameLengthChange(20, 40), + kFl20msTo40msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(40, 20), + kFl40msTo20msBandwidthBps}}; +} + +std::map +CreateChangeCriteriaFor20ms60msAnd120ms() { + return std::map{ + {FrameLengthController::Config::FrameLengthChange(20, 60), + kFl20msTo60msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 20), + kFl60msTo20msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 120), + kFl60msTo120msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(120, 60), + kFl120msTo60msBandwidthBps}}; +} + +std::map +CreateChangeCriteriaFor20ms40ms60msAnd120ms() { + return std::map{ + {FrameLengthController::Config::FrameLengthChange(20, 60), + kFl20msTo60msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 20), + kFl60msTo20msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(20, 40), + kFl20msTo40msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(40, 20), + kFl40msTo20msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(40, 60), + kFl40msTo60msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 40), + kFl60msTo40msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 120), + kFl60msTo120msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(120, 60), + kFl120msTo60msBandwidthBps}}; +} + +std::map +CreateChangeCriteriaFor40msAnd60ms() { + return std::map{ + {FrameLengthController::Config::FrameLengthChange(40, 60), + kFl40msTo60msBandwidthBps}, + {FrameLengthController::Config::FrameLengthChange(60, 40), + kFl60msTo40msBandwidthBps}}; +} + +void UpdateNetworkMetrics( + FrameLengthController* controller, + const absl::optional& uplink_bandwidth_bps, + const absl::optional& uplink_packet_loss_fraction, + const absl::optional& overhead_bytes_per_packet) { + // UpdateNetworkMetrics can accept multiple network metric updates at once. + // However, currently, the most used case is to update one metric at a time. + // To reflect this fact, we separate the calls. + if (uplink_bandwidth_bps) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller->UpdateNetworkMetrics(network_metrics); + } + if (uplink_packet_loss_fraction) { + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_packet_loss_fraction = uplink_packet_loss_fraction; + controller->UpdateNetworkMetrics(network_metrics); + } + if (overhead_bytes_per_packet) { + Controller::NetworkMetrics network_metrics; + network_metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + controller->UpdateNetworkMetrics(network_metrics); + } +} + +void CheckDecision(FrameLengthController* controller, + int expected_frame_length_ms) { + AudioEncoderRuntimeConfig config; + controller->MakeDecision(&config); + EXPECT_EQ(expected_frame_length_ms, config.frame_length_ms); +} + +} // namespace + +TEST(FrameLengthControllerTest, DecreaseTo20MsOnHighUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 60); + UpdateNetworkMetrics(controller.get(), kFl60msTo20msBandwidthBps, + absl::nullopt, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, DecreaseTo20MsOnHighUplinkPacketLossFraction) { + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 60); + UpdateNetworkMetrics(controller.get(), absl::nullopt, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, + Maintain60MsIf20MsNotInReceiverFrameLengthRange) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {60}, 60); + // Set FEC on that would cause frame length to decrease if receiver frame + // length range included 20ms. + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, IncreaseTo40MsOnMultipleConditions) { + // Increase to 40ms frame length if + // 1. `uplink_bandwidth_bps` is known to be smaller than a threshold AND + // 2. `uplink_packet_loss_fraction` is known to be smaller than a threshold + // AND + // 3. FEC is not decided or OFF. + auto controller = CreateController(CreateChangeCriteriaFor20msAnd40ms(), + kDefaultEncoderFrameLengthsMs, 20); + UpdateNetworkMetrics(controller.get(), kFl20msTo40msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 40); +} + +TEST(FrameLengthControllerTest, DecreaseTo40MsOnHighUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor40msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 40); + UpdateNetworkMetrics(controller.get(), kFl60msTo40msBandwidthBps, + absl::nullopt, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 40); +} + +TEST(FrameLengthControllerTest, Maintain60MsOnMultipleConditions) { + // Maintain 60ms frame length if + // 1. `uplink_bandwidth_bps` is at medium level, + // 2. `uplink_packet_loss_fraction` is at medium, + // 3. FEC is not decided ON. + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 60); + UpdateNetworkMetrics(controller.get(), kMediumBandwidthBps, + kMediumPacketLossFraction, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, IncreaseTo60MsOnMultipleConditions) { + // Increase to 60ms frame length if + // 1. `uplink_bandwidth_bps` is known to be smaller than a threshold AND + // 2. `uplink_packet_loss_fraction` is known to be smaller than a threshold + // AND + // 3. FEC is not decided or OFF. + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 20); + UpdateNetworkMetrics(controller.get(), kFl20msTo60msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, IncreaseTo60MsOnVeryLowUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 20); + // We set packet loss fraction to kFlDecreasingPacketLossFraction, which + // should have prevented frame length to increase, if the uplink bandwidth + // was not this low. + UpdateNetworkMetrics(controller.get(), VeryLowBitrate(20), + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, Maintain60MsOnVeryLowUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 60); + // We set packet loss fraction to FlDecreasingPacketLossFraction, which should + // have caused the frame length to decrease, if the uplink bandwidth was not + // this low. + UpdateNetworkMetrics(controller.get(), VeryLowBitrate(20), + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, UpdateMultipleNetworkMetricsAtOnce) { + // This test is similar to IncreaseTo60MsOnMultipleConditions. But instead of + // using ::UpdateNetworkMetrics(...), which calls + // FrameLengthController::UpdateNetworkMetrics(...) multiple times, we + // we call it only once. This is to verify that + // FrameLengthController::UpdateNetworkMetrics(...) can handle multiple + // network updates at once. This is, however, not a common use case in current + // audio_network_adaptor_impl.cc. + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 20); + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_bandwidth_bps = kFl20msTo60msBandwidthBps; + network_metrics.uplink_packet_loss_fraction = kFlIncreasingPacketLossFraction; + controller->UpdateNetworkMetrics(network_metrics); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, + Maintain20MsIf60MsNotInReceiverFrameLengthRange) { + auto controller = + CreateController(CreateChangeCriteriaFor20msAnd60ms(), {20}, 20); + // Use a low uplink bandwidth and a low uplink packet loss fraction that would + // cause frame length to increase if receiver frame length included 60ms. + UpdateNetworkMetrics(controller.get(), kFl20msTo60msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, Maintain20MsOnMediumUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 20); + UpdateNetworkMetrics(controller.get(), kMediumBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, Maintain20MsOnMediumUplinkPacketLossFraction) { + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 20); + // Use a low uplink bandwidth that would cause frame length to increase if + // uplink packet loss fraction was low. + UpdateNetworkMetrics(controller.get(), kFl20msTo60msBandwidthBps, + kMediumPacketLossFraction, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, Maintain60MsWhenNo120msCriteriaIsSet) { + auto controller = CreateController(CreateChangeCriteriaFor20msAnd60ms(), + kDefaultEncoderFrameLengthsMs, 60); + UpdateNetworkMetrics(controller.get(), kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, From120MsTo20MsOnHighUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + kDefaultEncoderFrameLengthsMs, 120); + // It takes two steps for frame length to go from 120ms to 20ms. + UpdateNetworkMetrics(controller.get(), kFl60msTo20msBandwidthBps, + absl::nullopt, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), kFl60msTo20msBandwidthBps, + absl::nullopt, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, From120MsTo20MsOnHighUplinkPacketLossFraction) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + kDefaultEncoderFrameLengthsMs, 120); + // It takes two steps for frame length to go from 120ms to 20ms. + UpdateNetworkMetrics(controller.get(), absl::nullopt, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), absl::nullopt, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +TEST(FrameLengthControllerTest, Maintain120MsOnVeryLowUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + kDefaultEncoderFrameLengthsMs, 120); + // We set packet loss fraction to FlDecreasingPacketLossFraction, which should + // have caused the frame length to decrease, if the uplink bandwidth was not + // this low. + UpdateNetworkMetrics(controller.get(), VeryLowBitrate(60), + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 120); +} + +TEST(FrameLengthControllerTest, From60MsTo120MsOnVeryLowUplinkBandwidth) { + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + kDefaultEncoderFrameLengthsMs, 60); + // We set packet loss fraction to FlDecreasingPacketLossFraction, which should + // have prevented frame length to increase, if the uplink bandwidth was not + // this low. + UpdateNetworkMetrics(controller.get(), VeryLowBitrate(60), + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 120); +} + +TEST(FrameLengthControllerTest, From20MsTo120MsOnMultipleConditions) { + // Increase to 120ms frame length if + // 1. `uplink_bandwidth_bps` is known to be smaller than a threshold AND + // 2. `uplink_packet_loss_fraction` is known to be smaller than a threshold. + auto controller = CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), + kDefaultEncoderFrameLengthsMs, 20); + // It takes two steps for frame length to go from 20ms to 120ms. + UpdateNetworkMetrics(controller.get(), kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + UpdateNetworkMetrics(controller.get(), kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 120); +} + +TEST(FrameLengthControllerTest, Stall60MsIf120MsNotInReceiverFrameLengthRange) { + auto controller = + CreateController(CreateChangeCriteriaFor20ms60msAnd120ms(), {20, 60}, 20); + UpdateNetworkMetrics(controller.get(), kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + UpdateNetworkMetrics(controller.get(), kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); +} + +TEST(FrameLengthControllerTest, CheckBehaviorOnChangingNetworkMetrics) { + auto controller = + CreateController(CreateChangeCriteriaFor20ms40ms60msAnd120ms(), + kDefaultEncoderFrameLengthsMs, 20); + UpdateNetworkMetrics(controller.get(), kMediumBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); + + UpdateNetworkMetrics(controller.get(), kFl20msTo40msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 40); + + UpdateNetworkMetrics(controller.get(), kFl60msTo40msBandwidthBps, + kMediumPacketLossFraction, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 40); + + UpdateNetworkMetrics(controller.get(), kFl20msTo60msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), kFl60msTo120msBandwidthBps, + kMediumPacketLossFraction, kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), kFl60msTo120msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 120); + + UpdateNetworkMetrics(controller.get(), kFl120msTo60msBandwidthBps, + kFlIncreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 60); + + UpdateNetworkMetrics(controller.get(), kFl60msTo40msBandwidthBps, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 40); + + UpdateNetworkMetrics(controller.get(), kMediumBandwidthBps, + kFlDecreasingPacketLossFraction, + kOverheadBytesPerPacket); + CheckDecision(controller.get(), 20); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc new file mode 100644 index 0000000000..36fc10ba82 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" + +#include + +#include "absl/algorithm/container.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +int OverheadBps(int overhead_bytes_per_packet, int frame_length_ms) { + return overhead_bytes_per_packet * 8 * 1000 / frame_length_ms; +} + +} // namespace + +FrameLengthControllerV2::FrameLengthControllerV2( + rtc::ArrayView encoder_frame_lengths_ms, + int min_payload_bitrate_bps, + bool use_slow_adaptation) + : encoder_frame_lengths_ms_(encoder_frame_lengths_ms.begin(), + encoder_frame_lengths_ms.end()), + min_payload_bitrate_bps_(min_payload_bitrate_bps), + use_slow_adaptation_(use_slow_adaptation) { + RTC_CHECK(!encoder_frame_lengths_ms_.empty()); + absl::c_sort(encoder_frame_lengths_ms_); +} + +void FrameLengthControllerV2::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.target_audio_bitrate_bps) { + target_bitrate_bps_ = network_metrics.target_audio_bitrate_bps; + } + if (network_metrics.overhead_bytes_per_packet) { + overhead_bytes_per_packet_ = network_metrics.overhead_bytes_per_packet; + } + if (network_metrics.uplink_bandwidth_bps) { + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; + } +} + +void FrameLengthControllerV2::MakeDecision(AudioEncoderRuntimeConfig* config) { + if (!target_bitrate_bps_ || !overhead_bytes_per_packet_ || + !uplink_bandwidth_bps_) { + return; + } + + auto it = + absl::c_find_if(encoder_frame_lengths_ms_, [&](int frame_length_ms) { + int target = use_slow_adaptation_ ? *uplink_bandwidth_bps_ + : *target_bitrate_bps_; + return target - + OverheadBps(*overhead_bytes_per_packet_, frame_length_ms) > + min_payload_bitrate_bps_; + }); + + // Longest frame length is chosen if none match our criteria. + config->frame_length_ms = it != encoder_frame_lengths_ms_.end() + ? *it + : encoder_frame_lengths_ms_.back(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h new file mode 100644 index 0000000000..d7102b0b44 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ + +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" + +namespace webrtc { + +class FrameLengthControllerV2 final : public Controller { + public: + FrameLengthControllerV2(rtc::ArrayView encoder_frame_lengths_ms, + int min_payload_bitrate_bps, + bool use_slow_adaptation); + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + std::vector encoder_frame_lengths_ms_; + const int min_payload_bitrate_bps_; + const bool use_slow_adaptation_; + + absl::optional uplink_bandwidth_bps_; + absl::optional target_bitrate_bps_; + absl::optional overhead_bytes_per_packet_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc new file mode 100644 index 0000000000..1c88f47c58 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" + +#include +#include + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr int kANASupportedFrameLengths[] = {20, 40, 60, 120}; +constexpr int kMinPayloadBitrateBps = 16000; + +} // namespace + +class FrameLengthControllerV2Test : public testing::Test { + protected: + AudioEncoderRuntimeConfig GetDecision() { + AudioEncoderRuntimeConfig config; + controller_->MakeDecision(&config); + return config; + } + + void SetOverhead(int overhead_bytes_per_packet) { + overhead_bytes_per_packet_ = overhead_bytes_per_packet; + Controller::NetworkMetrics metrics; + metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + controller_->UpdateNetworkMetrics(metrics); + } + + void SetTargetBitrate(int target_audio_bitrate_bps) { + target_audio_bitrate_bps_ = target_audio_bitrate_bps; + Controller::NetworkMetrics metrics; + metrics.target_audio_bitrate_bps = target_audio_bitrate_bps; + controller_->UpdateNetworkMetrics(metrics); + } + + void SetUplinkBandwidth(int uplink_bandwidth_bps) { + Controller::NetworkMetrics metrics; + metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller_->UpdateNetworkMetrics(metrics); + } + + void ExpectFrameLengthDecision(int expected_frame_length_ms) { + auto config = GetDecision(); + EXPECT_EQ(*config.frame_length_ms, expected_frame_length_ms); + } + + std::unique_ptr controller_ = + std::make_unique(kANASupportedFrameLengths, + kMinPayloadBitrateBps, + /*use_slow_adaptation=*/false); + absl::optional target_audio_bitrate_bps_; + absl::optional overhead_bytes_per_packet_; +}; + +// Don't return any decision if we haven't received all required network +// metrics. +TEST_F(FrameLengthControllerV2Test, RequireNetworkMetrics) { + auto config = GetDecision(); + EXPECT_FALSE(config.bitrate_bps); + EXPECT_FALSE(config.frame_length_ms); + + SetOverhead(30); + config = GetDecision(); + EXPECT_FALSE(config.frame_length_ms); + + SetTargetBitrate(32000); + config = GetDecision(); + EXPECT_FALSE(config.frame_length_ms); + + SetUplinkBandwidth(32000); + config = GetDecision(); + EXPECT_TRUE(config.frame_length_ms); +} + +TEST_F(FrameLengthControllerV2Test, UseFastAdaptation) { + SetOverhead(50); + SetTargetBitrate(50000); + SetUplinkBandwidth(50000); + ExpectFrameLengthDecision(20); + + SetTargetBitrate(20000); + ExpectFrameLengthDecision(120); + + SetTargetBitrate(30000); + ExpectFrameLengthDecision(40); + + SetTargetBitrate(25000); + ExpectFrameLengthDecision(60); +} + +TEST_F(FrameLengthControllerV2Test, UseSlowAdaptation) { + controller_ = std::make_unique( + kANASupportedFrameLengths, kMinPayloadBitrateBps, + /*use_slow_adaptation=*/true); + SetOverhead(50); + SetTargetBitrate(50000); + SetUplinkBandwidth(20000); + ExpectFrameLengthDecision(120); + + SetUplinkBandwidth(30000); + ExpectFrameLengthDecision(40); + + SetUplinkBandwidth(40000); + ExpectFrameLengthDecision(20); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h new file mode 100644 index 0000000000..346ed5db1a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_H_ + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_encoder.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" + +namespace webrtc { + +// An AudioNetworkAdaptor optimizes the audio experience by suggesting a +// suitable runtime configuration (bit rate, frame length, FEC, etc.) to the +// encoder based on network metrics. +class AudioNetworkAdaptor { + public: + virtual ~AudioNetworkAdaptor() = default; + + virtual void SetUplinkBandwidth(int uplink_bandwidth_bps) = 0; + + virtual void SetUplinkPacketLossFraction( + float uplink_packet_loss_fraction) = 0; + + virtual void SetRtt(int rtt_ms) = 0; + + virtual void SetTargetAudioBitrate(int target_audio_bitrate_bps) = 0; + + virtual void SetOverhead(size_t overhead_bytes_per_packet) = 0; + + virtual AudioEncoderRuntimeConfig GetEncoderRuntimeConfig() = 0; + + virtual void StartDebugDump(FILE* file_handle) = 0; + + virtual void StopDebugDump() = 0; + + virtual ANAStats GetStats() const = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h new file mode 100644 index 0000000000..bd16292f7e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_CONFIG_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_CONFIG_H_ + +#include + +#include "absl/types/optional.h" + +namespace webrtc { + +struct AudioEncoderRuntimeConfig { + AudioEncoderRuntimeConfig(); + AudioEncoderRuntimeConfig(const AudioEncoderRuntimeConfig& other); + ~AudioEncoderRuntimeConfig(); + AudioEncoderRuntimeConfig& operator=(const AudioEncoderRuntimeConfig& other); + bool operator==(const AudioEncoderRuntimeConfig& other) const; + absl::optional bitrate_bps; + absl::optional frame_length_ms; + // Note: This is what we tell the encoder. It doesn't have to reflect + // the actual NetworkMetrics; it's subject to our decision. + absl::optional uplink_packet_loss_fraction; + absl::optional enable_fec; + absl::optional enable_dtx; + + // Some encoders can encode fewer channels than the actual input to make + // better use of the bandwidth. `num_channels` sets the number of channels + // to encode. + absl::optional num_channels; + + // This is true if the last frame length change was an increase, and otherwise + // false. + // The value of this boolean is used to apply a different offset to the + // per-packet overhead that is reported by the BWE. The exact offset value + // is most important right after a frame length change, because the frame + // length change affects the overhead. In the steady state, the exact value is + // not important because the BWE will compensate. + bool last_fl_change_increase = false; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_INCLUDE_AUDIO_NETWORK_ADAPTOR_CONFIG_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h new file mode 100644 index 0000000000..26a9061745 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ + +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockAudioNetworkAdaptor : public AudioNetworkAdaptor { + public: + ~MockAudioNetworkAdaptor() override { Die(); } + MOCK_METHOD(void, Die, ()); + + MOCK_METHOD(void, SetUplinkBandwidth, (int uplink_bandwidth_bps), (override)); + + MOCK_METHOD(void, + SetUplinkPacketLossFraction, + (float uplink_packet_loss_fraction), + (override)); + + MOCK_METHOD(void, SetRtt, (int rtt_ms), (override)); + + MOCK_METHOD(void, + SetTargetAudioBitrate, + (int target_audio_bitrate_bps), + (override)); + + MOCK_METHOD(void, + SetOverhead, + (size_t overhead_bytes_per_packet), + (override)); + + MOCK_METHOD(AudioEncoderRuntimeConfig, + GetEncoderRuntimeConfig, + (), + (override)); + + MOCK_METHOD(void, StartDebugDump, (FILE * file_handle), (override)); + + MOCK_METHOD(void, StopDebugDump, (), (override)); + + MOCK_METHOD(ANAStats, GetStats, (), (const, override)); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h new file mode 100644 index 0000000000..de554c0517 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_H_ + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockController : public Controller { + public: + ~MockController() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, + UpdateNetworkMetrics, + (const NetworkMetrics& network_metrics), + (override)); + MOCK_METHOD(void, + MakeDecision, + (AudioEncoderRuntimeConfig * config), + (override)); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h new file mode 100644 index 0000000000..9e2fa466fc --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_MANAGER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_MANAGER_H_ + +#include + +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockControllerManager : public ControllerManager { + public: + ~MockControllerManager() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(std::vector, + GetSortedControllers, + (const Controller::NetworkMetrics& metrics), + (override)); + MOCK_METHOD(std::vector, GetControllers, (), (const, override)); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_MANAGER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h new file mode 100644 index 0000000000..0c6a9efe1d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_DEBUG_DUMP_WRITER_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_DEBUG_DUMP_WRITER_H_ + +#include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDebugDumpWriter : public DebugDumpWriter { + public: + ~MockDebugDumpWriter() override { Die(); } + MOCK_METHOD(void, Die, ()); + + MOCK_METHOD(void, + DumpEncoderRuntimeConfig, + (const AudioEncoderRuntimeConfig& config, int64_t timestamp), + (override)); + MOCK_METHOD(void, + DumpNetworkMetrics, + (const Controller::NetworkMetrics& metrics, int64_t timestamp), + (override)); +#if WEBRTC_ENABLE_PROTOBUF + MOCK_METHOD(void, + DumpControllerManagerConfig, + (const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp), + (override)); +#endif +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_DEBUG_DUMP_WRITER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/parse_ana_dump.py b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/parse_ana_dump.py new file mode 100755 index 0000000000..9c07c18c84 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/parse_ana_dump.py @@ -0,0 +1,149 @@ +#!/usr/bin/python2 +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# To run this script please copy "out///pyproto/webrtc/modules/ +# audio_coding/audio_network_adaptor/debug_dump_pb2.py" to this folder. +# The you can run this script with: +# "python parse_ana_dump.py -m uplink_bandwidth_bps -f dump_file.dat" +# You can add as may metrics or decisions to the plot as you like. +# form more information call: +# "python parse_ana_dump.py --help" + +import struct +from optparse import OptionParser + +import matplotlib.pyplot as plt + +import debug_dump_pb2 + + +def GetNextMessageSize(file_to_parse): + data = file_to_parse.read(4) + if data == '': + return 0 + return struct.unpack(' bandwidth + // + // If either a.x == b.x or a.y == b.y, the curve can be defined + // by a single point. (We merge the two points into one - either the lower or + // the leftmost one - for easier treatment.) + // + // y-axis ^ | + // | | + // | | + // | | + // | P|__________ + // |---------------> bandwidth + ThresholdCurve(const Point& left, const Point& right) + : a(GetPoint(left, right, true)), + b(GetPoint(left, right, false)), + slope(b.x - a.x == 0.0f ? 0.0f : (b.y - a.y) / (b.x - a.x)), + offset(a.y - slope * a.x) { + // TODO(eladalon): We might want to introduce some numerical validations. + } + + ThresholdCurve(float a_x, float a_y, float b_x, float b_y) + : ThresholdCurve(Point{a_x, a_y}, Point{b_x, b_y}) {} + + // Checks if a point is strictly below the curve. + bool IsBelowCurve(const Point& p) const { + if (p.x < a.x) { + return true; + } else if (p.x == a.x) { + // In principle, we could merge this into the next else, but to avoid + // numerical errors, we treat it separately. + return p.y < a.y; + } else if (a.x < p.x && p.x < b.x) { + return p.y < offset + slope * p.x; + } else { // if (b.x <= p.x) + return p.y < b.y; + } + } + + // Checks if a point is strictly above the curve. + bool IsAboveCurve(const Point& p) const { + if (p.x <= a.x) { + return false; + } else if (a.x < p.x && p.x < b.x) { + return p.y > offset + slope * p.x; + } else { // if (b.x <= p.x) + return p.y > b.y; + } + } + + bool operator<=(const ThresholdCurve& rhs) const { + // This curve is <= the rhs curve if no point from this curve is + // above a corresponding point from the rhs curve. + return !IsBelowCurve(rhs.a) && !IsBelowCurve(rhs.b) && + !rhs.IsAboveCurve(a) && !rhs.IsAboveCurve(b); + } + + private: + static const Point& GetPoint(const Point& left, + const Point& right, + bool is_for_left) { + RTC_DCHECK_LE(left.x, right.x); + RTC_DCHECK_GE(left.y, right.y); + + // Same X-value or Y-value triggers merging both points to the + // lower and/or left of the two points, respectively. + if (left.x == right.x) { + return right; + } else if (left.y == right.y) { + return left; + } + + // If unmerged, boolean flag determines which of the points is desired. + return is_for_left ? left : right; + } + + const Point a; + const Point b; + const float slope; + const float offset; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_UTIL_THRESHOLD_CURVE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc new file mode 100644 index 0000000000..dc3aec0b18 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc @@ -0,0 +1,632 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" + +#include + +#include "test/gtest.h" + +// A threshold curve divides 2D space into three domains - below, on and above +// the threshold curve. +// The curve is defined by two points. Those points, P1 and P2, are ordered so +// that (P1.x <= P2.x && P1.y >= P2.y). +// The part of the curve which is between the two points is hereon referred +// to as the "segment". +// A "ray" extends from P1 directly upwards into infinity; that's the "vertical +// ray". Likewise, a "horizontal ray" extends from P2 directly rightwards. +// +// ^ | // +// | | vertical ray // +// | | // +// | | // +// | P1| // +// | \ // +// | \ segment // +// | \ // +// | \ horizontal ray // +// | P2 ------------------ // +// *---------------------------> // + +namespace webrtc { + +namespace { +enum RelativePosition { kBelow, kOn, kAbove }; + +void CheckRelativePosition(const ThresholdCurve& curve, + ThresholdCurve::Point point, + RelativePosition pos) { + RTC_CHECK(pos == kBelow || pos == kOn || pos == kAbove); + + EXPECT_EQ(pos == kBelow, curve.IsBelowCurve(point)); + EXPECT_EQ(pos == kAbove, curve.IsAboveCurve(point)); +} +} // namespace + +// Test that the curve correctly reports the below/above position of points, +// when the curve is a "normal" one - P1 and P2 are different in both their +// X and Y values. +TEST(ThresholdCurveTest, PointPositionToCommonCurve) { + // The points (P1-P2) define the curve. // + // All other points are above/below/on the curve. // + // // + // ^ // + // | | // + // | A F J R V // + // | | // + // | B P1 K S W // + // | \ // + // | \ // + // | \ L // + // | \ // + // | C G M T X // + // | \ // + // | N \ // + // | \ // + // | D H O P2--Y---------------- // + // | E I Q U Z // + // *----------------------------------> // + constexpr ThresholdCurve::Point p1{1000, 2000}; + constexpr ThresholdCurve::Point p2{2000, 1000}; + + RTC_CHECK_GT((p1.x + p2.x) / 2, p1.x); + RTC_CHECK_LT((p1.x + p2.x) / 2, p2.x); + RTC_CHECK_LT((p1.y + p2.y) / 2, p1.y); + RTC_CHECK_GT((p1.y + p2.y) / 2, p2.y); + + const ThresholdCurve curve(p1, p2); + + { + // All cases where the point lies to the left of P1. + constexpr float x = p1.x - 1; + CheckRelativePosition(curve, {x, p1.y + 1}, kBelow); // A + CheckRelativePosition(curve, {x, p1.y + 0}, kBelow); // B + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kBelow); // C + CheckRelativePosition(curve, {x, p2.y + 0}, kBelow); // D + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // E + } + + { + // All cases where the point has the same x-value as P1. + constexpr float x = p1.x; + CheckRelativePosition(curve, {x, p1.y + 1}, kOn); // F + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // P1 + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kBelow); // G + CheckRelativePosition(curve, {x, p2.y + 0}, kBelow); // H + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // I + } + + { + // To make sure we're really covering all of the cases, make sure that P1 + // and P2 were chosen so that L would really be below K, and O would really + // be below N. (This would not hold if the Y values are too close together.) + RTC_CHECK_LT(((p1.y + p2.y) / 2) + 1, p1.y); + RTC_CHECK_LT(p2.y, ((p1.y + p2.y) / 2) - 1); + + // All cases where the point's x-value is between P1 and P2. + constexpr float x = (p1.x + p2.x) / 2; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // J + CheckRelativePosition(curve, {x, p1.y + 0}, kAbove); // K + CheckRelativePosition(curve, {x, ((p1.y + p2.y) / 2) + 1}, kAbove); // L + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kOn); // M + CheckRelativePosition(curve, {x, ((p1.y + p2.y) / 2) - 1}, kBelow); // N + CheckRelativePosition(curve, {x, p2.y + 0}, kBelow); // O + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // Q + } + + { + // All cases where the point has the same x-value as P2. + constexpr float x = p2.x; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // R + CheckRelativePosition(curve, {x, p1.y + 0}, kAbove); // S + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kAbove); // T + CheckRelativePosition(curve, {x, p2.y + 0}, kOn); // P2 + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // U + } + + { + // All cases where the point lies to the right of P2. + constexpr float x = p2.x + 1; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // V + CheckRelativePosition(curve, {x, p1.y + 0}, kAbove); // W + CheckRelativePosition(curve, {x, (p1.y + p2.y) / 2}, kAbove); // X + CheckRelativePosition(curve, {x, p2.y + 0}, kOn); // Y + CheckRelativePosition(curve, {x, p2.y - 1}, kBelow); // Z + } +} + +// Test that the curve correctly reports the below/above position of points, +// when the curve is defined by two points with the same Y value. +TEST(ThresholdCurveTest, PointPositionToCurveWithHorizaontalSegment) { + // The points (P1-P2) define the curve. + // All other points are above/below/on the curve. + // + // ^ + // | | + // | | + // | A D F I K + // | | + // | | + // | B P1--G--P2-L-- + // | C E H J M + // *------------------> + + constexpr ThresholdCurve::Point p1{100, 200}; + constexpr ThresholdCurve::Point p2{p1.x + 1, p1.y}; + + RTC_CHECK_GT((p1.x + p2.x) / 2, p1.x); + RTC_CHECK_LT((p1.x + p2.x) / 2, p2.x); + + const ThresholdCurve curve(p1, p2); + + { + // All cases where the point lies to the left of P1. + constexpr float x = p1.x - 1; + CheckRelativePosition(curve, {x, p1.y + 1}, kBelow); // A + CheckRelativePosition(curve, {x, p1.y + 0}, kBelow); // B + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // C + } + + { + // All cases where the point has the same x-value as P1. + constexpr float x = p1.x; + CheckRelativePosition(curve, {x, p1.y + 1}, kOn); // D + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // P1 + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // E + } + + { + // All cases where the point's x-value is between P1 and P2. + constexpr float x = (p1.x + p2.x) / 2; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // F + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // G + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // H + } + + { + // All cases where the point has the same x-value as P2. + constexpr float x = p2.x; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // I + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // P2 + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // J + } + + { + // All cases where the point lies to the right of P2. + constexpr float x = p2.x + 1; + CheckRelativePosition(curve, {x, p1.y + 1}, kAbove); // K + CheckRelativePosition(curve, {x, p1.y + 0}, kOn); // L + CheckRelativePosition(curve, {x, p1.y - 1}, kBelow); // M + } +} + +// Test that the curve correctly reports the below/above position of points, +// when the curve is defined by two points with the same X value. +TEST(ThresholdCurveTest, PointPositionToCurveWithVerticalSegment) { + // The points (P1-P2) define the curve. + // All other points are above/below/on the curve. + // + // ^ + // | | + // | A B C + // | | + // | D P1 E + // | | + // | F G H + // | | + // | I P2--J------ + // | K L M + // *------------------> + + constexpr ThresholdCurve::Point p1{100, 200}; + constexpr ThresholdCurve::Point p2{p1.x, p1.y - 1}; + + constexpr float left = p1.x - 1; + constexpr float on = p1.x; + constexpr float right = p1.x + 1; + + RTC_CHECK_LT((p1.y + p2.y) / 2, p1.y); + RTC_CHECK_GT((p1.y + p2.y) / 2, p2.y); + + const ThresholdCurve curve(p1, p2); + + { + // All cases where the point lies above P1. + constexpr float y = p1.y + 1; + CheckRelativePosition(curve, {left, y}, kBelow); // A + CheckRelativePosition(curve, {on, y}, kOn); // B + CheckRelativePosition(curve, {right, y}, kAbove); // C + } + + { + // All cases where the point has the same y-value as P1. + constexpr float y = p1.y; + CheckRelativePosition(curve, {left, y}, kBelow); // D + CheckRelativePosition(curve, {on, y}, kOn); // P1 + CheckRelativePosition(curve, {right, y}, kAbove); // E + } + + { + // All cases where the point's y-value is between P1 and P2. + constexpr float y = (p1.y + p2.y) / 2; + CheckRelativePosition(curve, {left, y}, kBelow); // F + CheckRelativePosition(curve, {on, y}, kOn); // G + CheckRelativePosition(curve, {right, y}, kAbove); // H + } + + { + // All cases where the point has the same y-value as P2. + constexpr float y = p2.y; + CheckRelativePosition(curve, {left, y}, kBelow); // I + CheckRelativePosition(curve, {on, y}, kOn); // P2 + CheckRelativePosition(curve, {right, y}, kOn); // J + } + + { + // All cases where the point lies below P2. + constexpr float y = p2.y - 1; + CheckRelativePosition(curve, {left, y}, kBelow); // K + CheckRelativePosition(curve, {on, y}, kBelow); // L + CheckRelativePosition(curve, {right, y}, kBelow); // M + } +} + +// Test that the curve correctly reports the below/above position of points, +// when the curve is defined by two points which are identical. +TEST(ThresholdCurveTest, PointPositionCurveWithNullSegment) { + // The points (P1-P2) define the curve. + // All other points are above/below/on the curve. + // + // ^ + // | | + // | A D F + // | | + // | B P---G------ + // | C E H + // *------------------> + + constexpr ThresholdCurve::Point p{100, 200}; + + const ThresholdCurve curve(p, p); + + { + // All cases where the point lies to the left of P. + constexpr float x = p.x - 1; + CheckRelativePosition(curve, {x, p.y + 1}, kBelow); // A + CheckRelativePosition(curve, {x, p.y + 0}, kBelow); // B + CheckRelativePosition(curve, {x, p.y - 1}, kBelow); // C + } + + { + // All cases where the point has the same x-value as P. + constexpr float x = p.x + 0; + CheckRelativePosition(curve, {x, p.y + 1}, kOn); // D + CheckRelativePosition(curve, {x, p.y + 0}, kOn); // P + CheckRelativePosition(curve, {x, p.y - 1}, kBelow); // E + } + + { + // All cases where the point lies to the right of P. + constexpr float x = p.x + 1; + CheckRelativePosition(curve, {x, p.y + 1}, kAbove); // F + CheckRelativePosition(curve, {x, p.y + 0}, kOn); // G + CheckRelativePosition(curve, {x, p.y - 1}, kBelow); // H + } +} + +// Test that the relative position of two curves is computed correctly when +// the two curves have the same projection on the X-axis. +TEST(ThresholdCurveTest, TwoCurvesSegmentHasSameProjectionAxisX) { + // ^ // + // | C1 + C2 // + // | | // + // | |\ // + // | | \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ -------- C2 // + // | --------- C1 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + // Same x-values, but higher on Y. (Can be parallel, but doesn't have to be.) + constexpr ThresholdCurve::Point c2_left{c1_left.x, c1_left.y + 20}; + constexpr ThresholdCurve::Point c2_right{c1_right.x, c1_right.y + 10}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the higher curve's projection on the X-axis is a strict subset of the +// lower curve's projection on the X-axis (on both ends). +TEST(ThresholdCurveTest, TwoCurvesSegmentOfHigherSubsetProjectionAxisX) { + // ^ // + // | C1 C2 // + // | | | // + // | | | // + // | \ | // + // | \ | // + // | \ \ // + // | \ \ // + // | \ --------- C2 // + // | \ // + // | \ // + // | ---------C1 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{6, 11}; + constexpr ThresholdCurve::Point c2_right{9, 7}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the higher curve's right point is above lower curve's horizontal ray (meaning +// the higher curve's projection on the X-axis extends further right than +// the lower curve's). +TEST(ThresholdCurveTest, + TwoCurvesRightPointOfHigherCurveAboveHorizontalRayOfLower) { + // ^ // + // | C1 + C2 // + // | | // + // | |\ // + // | | \ // + // | | \ // + // | | \ // + // | | \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ ----- C2 // + // | --------- C1 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{c1_left.x, c1_left.y + 1}; + constexpr ThresholdCurve::Point c2_right{c1_right.x + 1, c1_right.y + 1}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the higher curve's points are on the lower curve's rays (left point on the +// veritcal ray, right point on the horizontal ray). +TEST(ThresholdCurveTest, TwoCurvesPointsOfHigherOnRaysOfLower) { + // ^ + // | C1 + C2 // + // | | // + // | |\ // + // | | \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | ----- C1 + C2 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + // Same x-values, but one of the points is higher on Y (the other isn't). + constexpr ThresholdCurve::Point c2_left{c1_left.x, c1_left.y + 2}; + constexpr ThresholdCurve::Point c2_right{c1_right.x + 3, c1_right.y}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the second curve's segment intersects the first curve's vertical ray. +TEST(ThresholdCurveTest, SecondCurveCrossesVerticalRayOfFirstCurve) { + // ^ // + // | C2 C1 // + // | | | // + // | \| // + // | | // + // | |\ // + // | | \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ ------- C2 // + // | -------- C1 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{c1_left.x - 1, c1_left.y + 1}; + constexpr ThresholdCurve::Point c2_right{c1_right.x, c1_right.y + 1}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the second curve's segment intersects the first curve's horizontal ray. +TEST(ThresholdCurveTest, SecondCurveCrossesHorizontalRayOfFirstCurve) { + // ^ // + // | C1 + C2 // + // | | // + // | |\ // + // | \ \ // + // | \ \ // + // | \ \ // + // | \ \ // + // | ----------- C1 // + // | \ // + // | ------- C2 // + // *--------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{c1_left.x, c1_left.y + 1}; + constexpr ThresholdCurve::Point c2_right{c1_right.x + 2, c1_right.y - 1}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// the second curve's segment intersects the first curve's segment. +TEST(ThresholdCurveTest, TwoCurvesWithCrossingSegments) { + // ^ // + // | C2 C1 // + // | | | // + // | | | // + // | | \ // + // | | \ // + // | -_ \ // + // | -_ \ // + // | -_\ // + // | -_ // + // | \-_ // + // | \ ---------- C2 // + // | ----------- C1 // + // | // + // | // + // *-------------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_right); + + constexpr ThresholdCurve::Point c2_left{4, 9}; + constexpr ThresholdCurve::Point c2_right{10, 6}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + // The test is structured so that the two curves intersect at (8, 7). + RTC_CHECK(!c1_curve.IsAboveCurve({8, 7})); + RTC_CHECK(!c1_curve.IsBelowCurve({8, 7})); + RTC_CHECK(!c2_curve.IsAboveCurve({8, 7})); + RTC_CHECK(!c2_curve.IsBelowCurve({8, 7})); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_FALSE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// both curves are identical. +TEST(ThresholdCurveTest, IdenticalCurves) { + // ^ // + // | C1 + C2 // + // | | // + // | | // + // | \ // + // | \ // + // | \ // + // | ------- C1 + C2 // + // *---------------------> // + + constexpr ThresholdCurve::Point left{5, 10}; + constexpr ThresholdCurve::Point right{10, 5}; + + const ThresholdCurve c1_curve(left, right); + const ThresholdCurve c2_curve(left, right); + + EXPECT_TRUE(c1_curve <= c2_curve); + EXPECT_TRUE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// they are "nearly identical" - the first curve's segment is contained within +// the second curve's segment, but the second curve's segment extends further +// to the left (which also produces separate vertical rays for the curves). +TEST(ThresholdCurveTest, NearlyIdenticalCurvesSecondContinuesOnOtherLeftSide) { + // ^ // + // | C2 C1 // + // | | | // + // | | | // + // | \| // + // | | // + // | \ // + // | \ // + // | \ // + // | ----- C1 + C2 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_left); + + constexpr ThresholdCurve::Point c2_left{c1_left.x - 1, c1_left.y + 1}; + constexpr ThresholdCurve::Point c2_right = c1_right; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_TRUE(c2_curve <= c1_curve); +} + +// Test that the relative position of two curves is computed correctly when +// they are "nearly identical" - the first curve's segment is contained within +// the second curve's segment, but the second curve's segment extends further +// to the right (which also produces separate horizontal rays for the curves). +TEST(ThresholdCurveTest, NearlyIdenticalCurvesSecondContinuesOnOtherRightSide) { + // ^ // + // | C1 + C2 // + // | | // + // | | // + // | \ // + // | \ // + // | \ // + // | \----------- C1 // + // | \ // + // | ---------- C2 // + // *---------------------> // + + constexpr ThresholdCurve::Point c1_left{5, 10}; + constexpr ThresholdCurve::Point c1_right{10, 5}; + const ThresholdCurve c1_curve(c1_left, c1_left); + + constexpr ThresholdCurve::Point c2_left = c1_left; + constexpr ThresholdCurve::Point c2_right{c1_right.x + 1, c1_right.y - 1}; + const ThresholdCurve c2_curve(c2_left, c2_right); + + EXPECT_FALSE(c1_curve <= c2_curve); + EXPECT_TRUE(c2_curve <= c1_curve); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// The higher-left point must be given as the first point, and the lower-right +// point must be given as the second. +// This necessarily produces a non-positive slope. +TEST(ThresholdCurveDeathTest, WrongOrderPoints) { + std::unique_ptr curve; + constexpr ThresholdCurve::Point left{5, 10}; + constexpr ThresholdCurve::Point right{10, 5}; + EXPECT_DEATH(curve.reset(new ThresholdCurve(right, left)), ""); +} +#endif + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor_config_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor_config_gn/moz.build new file mode 100644 index 0000000000..37cf81ad9f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor_config_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_config.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("audio_network_adaptor_config_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor_gn/moz.build new file mode 100644 index 0000000000..1d0905ebc1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor_gn/moz.build @@ -0,0 +1,242 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/bitrate_controller.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/channel_controller.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/dtx_controller.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/event_log_writer.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller.cc", + "/third_party/libwebrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("audio_network_adaptor_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/audio_decoder.h b/third_party/libwebrtc/modules/audio_coding/codecs/audio_decoder.h new file mode 100644 index 0000000000..b7b15cdd6e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/audio_decoder.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is for backwards compatibility only! Use +// webrtc/api/audio_codecs/audio_decoder.h instead! +// TODO(kwiberg): Remove it. + +#ifndef MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ +#define MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ + +#include "api/audio_codecs/audio_decoder.h" + +#endif // MODULES_AUDIO_CODING_CODECS_AUDIO_DECODER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/audio_encoder.h b/third_party/libwebrtc/modules/audio_coding/codecs/audio_encoder.h new file mode 100644 index 0000000000..010ae6705f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/audio_encoder.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is for backwards compatibility only! Use +// webrtc/api/audio_codecs/audio_encoder.h instead! +// TODO(ossu): Remove it. + +#ifndef MODULES_AUDIO_CODING_CODECS_AUDIO_ENCODER_H_ +#define MODULES_AUDIO_CODING_CODECS_AUDIO_ENCODER_H_ + +#include "api/audio_codecs/audio_encoder.h" + +#endif // MODULES_AUDIO_CODING_CODECS_AUDIO_ENCODER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc new file mode 100644 index 0000000000..4a2b261a59 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" + +#include + +#include "test/gtest.h" + +namespace webrtc { + +TEST(AudioDecoderFactoryTest, CreateUnknownDecoder) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("rey", 8000, 1), absl::nullopt)); +} + +TEST(AudioDecoderFactoryTest, CreatePcmu) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // PCMu supports 8 kHz, and any number of channels. + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 8000, 0), absl::nullopt)); + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 8000, 1), absl::nullopt)); + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 8000, 2), absl::nullopt)); + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 8000, 3), absl::nullopt)); + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("pcmu", 16000, 1), absl::nullopt)); +} + +TEST(AudioDecoderFactoryTest, CreatePcma) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // PCMa supports 8 kHz, and any number of channels. + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("pcma", 8000, 0), absl::nullopt)); + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("pcma", 8000, 1), absl::nullopt)); + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("pcma", 8000, 2), absl::nullopt)); + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("pcma", 8000, 3), absl::nullopt)); + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("pcma", 16000, 1), absl::nullopt)); +} + +TEST(AudioDecoderFactoryTest, CreateIlbc) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // iLBC supports 8 kHz, 1 channel. + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("ilbc", 8000, 0), absl::nullopt)); +#ifdef WEBRTC_CODEC_ILBC + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("ilbc", 8000, 1), absl::nullopt)); +#endif + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("ilbc", 8000, 2), absl::nullopt)); + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("ilbc", 16000, 1), absl::nullopt)); +} + +TEST(AudioDecoderFactoryTest, CreateL16) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // L16 supports any clock rate and any number of channels up to 24. + const int clockrates[] = {8000, 16000, 32000, 48000}; + const int num_channels[] = {1, 2, 3, 24}; + for (int clockrate : clockrates) { + EXPECT_FALSE(adf->MakeAudioDecoder(SdpAudioFormat("l16", clockrate, 0), + absl::nullopt)); + for (int channels : num_channels) { + EXPECT_TRUE(adf->MakeAudioDecoder( + SdpAudioFormat("l16", clockrate, channels), absl::nullopt)); + } + } +} + +// Tests that using more channels than the maximum does not work +TEST(AudioDecoderFactoryTest, MaxNrOfChannels) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + std::vector codecs = { +#ifdef WEBRTC_CODEC_OPUS + "opus", +#endif +#ifdef WEBRTC_CODEC_ILBC + "ilbc", +#endif + "pcmu", + "pcma", + "l16", + "G722", + "G711", + }; + + for (auto codec : codecs) { + EXPECT_FALSE(adf->MakeAudioDecoder( + SdpAudioFormat(codec, 32000, AudioDecoder::kMaxNumberOfChannels + 1), + absl::nullopt)); + } +} + +TEST(AudioDecoderFactoryTest, CreateG722) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // g722 supports 8 kHz, 1-2 channels. + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 0), absl::nullopt)); + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 1), absl::nullopt)); + EXPECT_TRUE( + adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 2), absl::nullopt)); + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 3), absl::nullopt)); + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("g722", 16000, 1), absl::nullopt)); + EXPECT_FALSE( + adf->MakeAudioDecoder(SdpAudioFormat("g722", 32000, 1), absl::nullopt)); + + // g722 actually uses a 16 kHz sample rate instead of the nominal 8 kHz. + std::unique_ptr dec = + adf->MakeAudioDecoder(SdpAudioFormat("g722", 8000, 1), absl::nullopt); + EXPECT_EQ(16000, dec->SampleRateHz()); +} + +TEST(AudioDecoderFactoryTest, CreateOpus) { + rtc::scoped_refptr adf = + CreateBuiltinAudioDecoderFactory(); + ASSERT_TRUE(adf); + // Opus supports 48 kHz, 2 channels, and wants a "stereo" parameter whose + // value is either "0" or "1". + for (int hz : {8000, 16000, 32000, 48000}) { + for (int channels : {0, 1, 2, 3}) { + for (std::string stereo : {"XX", "0", "1", "2"}) { + SdpAudioFormat::Parameters params; + if (stereo != "XX") { + params["stereo"] = stereo; + } + const bool good = (hz == 48000 && channels == 2 && + (stereo == "XX" || stereo == "0" || stereo == "1")); + EXPECT_EQ(good, + static_cast(adf->MakeAudioDecoder( + SdpAudioFormat("opus", hz, channels, std::move(params)), + absl::nullopt))); + } + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory_unittest.cc new file mode 100644 index 0000000000..26ae1eda8a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory_unittest.cc @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/builtin_audio_encoder_factory.h" + +#include +#include +#include + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +class AudioEncoderFactoryTest + : public ::testing::TestWithParam> { +}; + +TEST_P(AudioEncoderFactoryTest, SupportsAtLeastOneFormat) { + auto factory = GetParam(); + auto supported_encoders = factory->GetSupportedEncoders(); + EXPECT_FALSE(supported_encoders.empty()); +} + +TEST_P(AudioEncoderFactoryTest, CanQueryAllSupportedFormats) { + auto factory = GetParam(); + auto supported_encoders = factory->GetSupportedEncoders(); + for (const auto& spec : supported_encoders) { + auto info = factory->QueryAudioEncoder(spec.format); + EXPECT_TRUE(info); + } +} + +TEST_P(AudioEncoderFactoryTest, CanConstructAllSupportedEncoders) { + auto factory = GetParam(); + auto supported_encoders = factory->GetSupportedEncoders(); + for (const auto& spec : supported_encoders) { + auto info = factory->QueryAudioEncoder(spec.format); + auto encoder = factory->MakeAudioEncoder(127, spec.format, absl::nullopt); + EXPECT_TRUE(encoder); + EXPECT_EQ(encoder->SampleRateHz(), info->sample_rate_hz); + EXPECT_EQ(encoder->NumChannels(), info->num_channels); + EXPECT_EQ(encoder->RtpTimestampRateHz(), spec.format.clockrate_hz); + } +} + +TEST_P(AudioEncoderFactoryTest, CanRunAllSupportedEncoders) { + constexpr int kTestPayloadType = 127; + auto factory = GetParam(); + auto supported_encoders = factory->GetSupportedEncoders(); + for (const auto& spec : supported_encoders) { + auto encoder = + factory->MakeAudioEncoder(kTestPayloadType, spec.format, absl::nullopt); + EXPECT_TRUE(encoder); + encoder->Reset(); + const int num_samples = rtc::checked_cast( + encoder->SampleRateHz() * encoder->NumChannels() / 100); + rtc::Buffer out; + rtc::BufferT audio; + audio.SetData(num_samples, [](rtc::ArrayView audio) { + for (size_t i = 0; i != audio.size(); ++i) { + // Just put some numbers in there, ensure they're within range. + audio[i] = + static_cast(i & std::numeric_limits::max()); + } + return audio.size(); + }); + // This is here to stop the test going forever with a broken encoder. + constexpr int kMaxEncodeCalls = 100; + int blocks = 0; + for (; blocks < kMaxEncodeCalls; ++blocks) { + AudioEncoder::EncodedInfo info = encoder->Encode( + blocks * encoder->RtpTimestampRateHz() / 100, audio, &out); + EXPECT_EQ(info.encoded_bytes, out.size()); + if (info.encoded_bytes > 0) { + EXPECT_EQ(0u, info.encoded_timestamp); + EXPECT_EQ(kTestPayloadType, info.payload_type); + break; + } + } + ASSERT_LT(blocks, kMaxEncodeCalls); + const unsigned int next_timestamp = + blocks * encoder->RtpTimestampRateHz() / 100; + out.Clear(); + for (; blocks < kMaxEncodeCalls; ++blocks) { + AudioEncoder::EncodedInfo info = encoder->Encode( + blocks * encoder->RtpTimestampRateHz() / 100, audio, &out); + EXPECT_EQ(info.encoded_bytes, out.size()); + if (info.encoded_bytes > 0) { + EXPECT_EQ(next_timestamp, info.encoded_timestamp); + EXPECT_EQ(kTestPayloadType, info.payload_type); + break; + } + } + ASSERT_LT(blocks, kMaxEncodeCalls); + } +} + +INSTANTIATE_TEST_SUITE_P(BuiltinAudioEncoderFactoryTest, + AudioEncoderFactoryTest, + ::testing::Values(CreateBuiltinAudioEncoderFactory())); + +TEST(BuiltinAudioEncoderFactoryTest, SupportsTheExpectedFormats) { + using ::testing::ElementsAreArray; + // Check that we claim to support the formats we expect from build flags, and + // we've ordered them correctly. + auto factory = CreateBuiltinAudioEncoderFactory(); + auto specs = factory->GetSupportedEncoders(); + + const std::vector supported_formats = [&specs] { + std::vector formats; + formats.reserve(specs.size()); + for (const auto& spec : specs) { + formats.push_back(spec.format); + } + return formats; + }(); + + const std::vector expected_formats = { +#ifdef WEBRTC_CODEC_OPUS + {"opus", 48000, 2, {{"minptime", "10"}, {"useinbandfec", "1"}}}, +#endif +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) + {"isac", 16000, 1}, +#endif +#ifdef WEBRTC_CODEC_ISAC + {"isac", 32000, 1}, +#endif + {"G722", 8000, 1}, +#ifdef WEBRTC_CODEC_ILBC + {"ilbc", 8000, 1}, +#endif + {"pcmu", 8000, 1}, + {"pcma", 8000, 1} + }; + + ASSERT_THAT(supported_formats, ElementsAreArray(expected_formats)); +} + +// Tests that using more channels than the maximum does not work. +TEST(BuiltinAudioEncoderFactoryTest, MaxNrOfChannels) { + rtc::scoped_refptr aef = + CreateBuiltinAudioEncoderFactory(); + std::vector codecs = { +#ifdef WEBRTC_CODEC_OPUS + "opus", +#endif +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) + "isac", +#endif +#ifdef WEBRTC_CODEC_ILBC + "ilbc", +#endif + "pcmu", + "pcma", + "l16", + "G722", + "G711", + }; + + for (auto codec : codecs) { + EXPECT_FALSE(aef->MakeAudioEncoder( + /*payload_type=*/111, + /*format=*/ + SdpAudioFormat(codec, 32000, AudioEncoder::kMaxNumberOfChannels + 1), + /*codec_pair_id=*/absl::nullopt)); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc b/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc new file mode 100644 index 0000000000..7546ac178f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/units/time_delta.h" +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { + +const int kMaxFrameSizeMs = 60; + +class AudioEncoderCng final : public AudioEncoder { + public: + explicit AudioEncoderCng(AudioEncoderCngConfig&& config); + ~AudioEncoderCng() override; + + // Not copyable or moveable. + AudioEncoderCng(const AudioEncoderCng&) = delete; + AudioEncoderCng(AudioEncoderCng&&) = delete; + AudioEncoderCng& operator=(const AudioEncoderCng&) = delete; + AudioEncoderCng& operator=(AudioEncoderCng&&) = delete; + + int SampleRateHz() const override; + size_t NumChannels() const override; + int RtpTimestampRateHz() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + void Reset() override; + bool SetFec(bool enable) override; + bool SetDtx(bool enable) override; + bool SetApplication(Application application) override; + void SetMaxPlaybackRate(int frequency_hz) override; + rtc::ArrayView> ReclaimContainedEncoders() + override; + void OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional bwe_period_ms) override; + absl::optional> GetFrameLengthRange() + const override; + + private: + EncodedInfo EncodePassive(size_t frames_to_encode, rtc::Buffer* encoded); + EncodedInfo EncodeActive(size_t frames_to_encode, rtc::Buffer* encoded); + size_t SamplesPer10msFrame() const; + + std::unique_ptr speech_encoder_; + const int cng_payload_type_; + const int num_cng_coefficients_; + const int sid_frame_interval_ms_; + std::vector speech_buffer_; + std::vector rtp_timestamps_; + bool last_frame_active_; + std::unique_ptr vad_; + std::unique_ptr cng_encoder_; +}; + +AudioEncoderCng::AudioEncoderCng(AudioEncoderCngConfig&& config) + : speech_encoder_((static_cast([&] { + RTC_CHECK(config.IsOk()) << "Invalid configuration."; + }()), + std::move(config.speech_encoder))), + cng_payload_type_(config.payload_type), + num_cng_coefficients_(config.num_cng_coefficients), + sid_frame_interval_ms_(config.sid_frame_interval_ms), + last_frame_active_(true), + vad_(config.vad ? std::unique_ptr(config.vad) + : CreateVad(config.vad_mode)), + cng_encoder_(new ComfortNoiseEncoder(SampleRateHz(), + sid_frame_interval_ms_, + num_cng_coefficients_)) {} + +AudioEncoderCng::~AudioEncoderCng() = default; + +int AudioEncoderCng::SampleRateHz() const { + return speech_encoder_->SampleRateHz(); +} + +size_t AudioEncoderCng::NumChannels() const { + return 1; +} + +int AudioEncoderCng::RtpTimestampRateHz() const { + return speech_encoder_->RtpTimestampRateHz(); +} + +size_t AudioEncoderCng::Num10MsFramesInNextPacket() const { + return speech_encoder_->Num10MsFramesInNextPacket(); +} + +size_t AudioEncoderCng::Max10MsFramesInAPacket() const { + return speech_encoder_->Max10MsFramesInAPacket(); +} + +int AudioEncoderCng::GetTargetBitrate() const { + return speech_encoder_->GetTargetBitrate(); +} + +AudioEncoder::EncodedInfo AudioEncoderCng::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + const size_t samples_per_10ms_frame = SamplesPer10msFrame(); + RTC_CHECK_EQ(speech_buffer_.size(), + rtp_timestamps_.size() * samples_per_10ms_frame); + rtp_timestamps_.push_back(rtp_timestamp); + RTC_DCHECK_EQ(samples_per_10ms_frame, audio.size()); + speech_buffer_.insert(speech_buffer_.end(), audio.cbegin(), audio.cend()); + const size_t frames_to_encode = speech_encoder_->Num10MsFramesInNextPacket(); + if (rtp_timestamps_.size() < frames_to_encode) { + return EncodedInfo(); + } + RTC_CHECK_LE(frames_to_encode * 10, kMaxFrameSizeMs) + << "Frame size cannot be larger than " << kMaxFrameSizeMs + << " ms when using VAD/CNG."; + + // Group several 10 ms blocks per VAD call. Call VAD once or twice using the + // following split sizes: + // 10 ms = 10 + 0 ms; 20 ms = 20 + 0 ms; 30 ms = 30 + 0 ms; + // 40 ms = 20 + 20 ms; 50 ms = 30 + 20 ms; 60 ms = 30 + 30 ms. + size_t blocks_in_first_vad_call = + (frames_to_encode > 3 ? 3 : frames_to_encode); + if (frames_to_encode == 4) + blocks_in_first_vad_call = 2; + RTC_CHECK_GE(frames_to_encode, blocks_in_first_vad_call); + const size_t blocks_in_second_vad_call = + frames_to_encode - blocks_in_first_vad_call; + + // Check if all of the buffer is passive speech. Start with checking the first + // block. + Vad::Activity activity = vad_->VoiceActivity( + &speech_buffer_[0], samples_per_10ms_frame * blocks_in_first_vad_call, + SampleRateHz()); + if (activity == Vad::kPassive && blocks_in_second_vad_call > 0) { + // Only check the second block if the first was passive. + activity = vad_->VoiceActivity( + &speech_buffer_[samples_per_10ms_frame * blocks_in_first_vad_call], + samples_per_10ms_frame * blocks_in_second_vad_call, SampleRateHz()); + } + + EncodedInfo info; + switch (activity) { + case Vad::kPassive: { + info = EncodePassive(frames_to_encode, encoded); + last_frame_active_ = false; + break; + } + case Vad::kActive: { + info = EncodeActive(frames_to_encode, encoded); + last_frame_active_ = true; + break; + } + default: { + RTC_CHECK_NOTREACHED(); + } + } + + speech_buffer_.erase( + speech_buffer_.begin(), + speech_buffer_.begin() + frames_to_encode * samples_per_10ms_frame); + rtp_timestamps_.erase(rtp_timestamps_.begin(), + rtp_timestamps_.begin() + frames_to_encode); + return info; +} + +void AudioEncoderCng::Reset() { + speech_encoder_->Reset(); + speech_buffer_.clear(); + rtp_timestamps_.clear(); + last_frame_active_ = true; + vad_->Reset(); + cng_encoder_.reset(new ComfortNoiseEncoder( + SampleRateHz(), sid_frame_interval_ms_, num_cng_coefficients_)); +} + +bool AudioEncoderCng::SetFec(bool enable) { + return speech_encoder_->SetFec(enable); +} + +bool AudioEncoderCng::SetDtx(bool enable) { + return speech_encoder_->SetDtx(enable); +} + +bool AudioEncoderCng::SetApplication(Application application) { + return speech_encoder_->SetApplication(application); +} + +void AudioEncoderCng::SetMaxPlaybackRate(int frequency_hz) { + speech_encoder_->SetMaxPlaybackRate(frequency_hz); +} + +rtc::ArrayView> +AudioEncoderCng::ReclaimContainedEncoders() { + return rtc::ArrayView>(&speech_encoder_, 1); +} + +void AudioEncoderCng::OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + speech_encoder_->OnReceivedUplinkPacketLossFraction( + uplink_packet_loss_fraction); +} + +void AudioEncoderCng::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional bwe_period_ms) { + speech_encoder_->OnReceivedUplinkBandwidth(target_audio_bitrate_bps, + bwe_period_ms); +} + +absl::optional> +AudioEncoderCng::GetFrameLengthRange() const { + return speech_encoder_->GetFrameLengthRange(); +} + +AudioEncoder::EncodedInfo AudioEncoderCng::EncodePassive( + size_t frames_to_encode, + rtc::Buffer* encoded) { + bool force_sid = last_frame_active_; + bool output_produced = false; + const size_t samples_per_10ms_frame = SamplesPer10msFrame(); + AudioEncoder::EncodedInfo info; + + for (size_t i = 0; i < frames_to_encode; ++i) { + // It's important not to pass &info.encoded_bytes directly to + // WebRtcCng_Encode(), since later loop iterations may return zero in + // that value, in which case we don't want to overwrite any value from + // an earlier iteration. + size_t encoded_bytes_tmp = + cng_encoder_->Encode(rtc::ArrayView( + &speech_buffer_[i * samples_per_10ms_frame], + samples_per_10ms_frame), + force_sid, encoded); + + if (encoded_bytes_tmp > 0) { + RTC_CHECK(!output_produced); + info.encoded_bytes = encoded_bytes_tmp; + output_produced = true; + force_sid = false; + } + } + + info.encoded_timestamp = rtp_timestamps_.front(); + info.payload_type = cng_payload_type_; + info.send_even_if_empty = true; + info.speech = false; + return info; +} + +AudioEncoder::EncodedInfo AudioEncoderCng::EncodeActive(size_t frames_to_encode, + rtc::Buffer* encoded) { + const size_t samples_per_10ms_frame = SamplesPer10msFrame(); + AudioEncoder::EncodedInfo info; + for (size_t i = 0; i < frames_to_encode; ++i) { + info = + speech_encoder_->Encode(rtp_timestamps_.front(), + rtc::ArrayView( + &speech_buffer_[i * samples_per_10ms_frame], + samples_per_10ms_frame), + encoded); + if (i + 1 == frames_to_encode) { + RTC_CHECK_GT(info.encoded_bytes, 0) << "Encoder didn't deliver data."; + } else { + RTC_CHECK_EQ(info.encoded_bytes, 0) + << "Encoder delivered data too early."; + } + } + return info; +} + +size_t AudioEncoderCng::SamplesPer10msFrame() const { + return rtc::CheckedDivExact(10 * SampleRateHz(), 1000); +} + +} // namespace + +AudioEncoderCngConfig::AudioEncoderCngConfig() = default; +AudioEncoderCngConfig::AudioEncoderCngConfig(AudioEncoderCngConfig&&) = default; +AudioEncoderCngConfig::~AudioEncoderCngConfig() = default; + +bool AudioEncoderCngConfig::IsOk() const { + if (num_channels != 1) + return false; + if (!speech_encoder) + return false; + if (num_channels != speech_encoder->NumChannels()) + return false; + if (sid_frame_interval_ms < + static_cast(speech_encoder->Max10MsFramesInAPacket() * 10)) + return false; + if (num_cng_coefficients > WEBRTC_CNG_MAX_LPC_ORDER || + num_cng_coefficients <= 0) + return false; + return true; +} + +std::unique_ptr CreateComfortNoiseEncoder( + AudioEncoderCngConfig&& config) { + return std::make_unique(std::move(config)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h b/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h new file mode 100644 index 0000000000..8a1183489f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_CNG_AUDIO_ENCODER_CNG_H_ +#define MODULES_AUDIO_CODING_CODECS_CNG_AUDIO_ENCODER_CNG_H_ + +#include + +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "common_audio/vad/include/vad.h" + +namespace webrtc { + +struct AudioEncoderCngConfig { + // Moveable, not copyable. + AudioEncoderCngConfig(); + AudioEncoderCngConfig(AudioEncoderCngConfig&&); + ~AudioEncoderCngConfig(); + + bool IsOk() const; + + size_t num_channels = 1; + int payload_type = 13; + std::unique_ptr speech_encoder; + Vad::Aggressiveness vad_mode = Vad::kVadNormal; + int sid_frame_interval_ms = 100; + int num_cng_coefficients = 8; + // The Vad pointer is mainly for testing. If a NULL pointer is passed, the + // AudioEncoderCng creates (and destroys) a Vad object internally. If an + // object is passed, the AudioEncoderCng assumes ownership of the Vad + // object. + Vad* vad = nullptr; +}; + +std::unique_ptr CreateComfortNoiseEncoder( + AudioEncoderCngConfig&& config); + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_CNG_AUDIO_ENCODER_CNG_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc new file mode 100644 index 0000000000..c688004363 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc @@ -0,0 +1,520 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" + +#include +#include + +#include "common_audio/vad/mock/mock_vad.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/mock_audio_encoder.h" +#include "test/testsupport/rtc_expect_death.h" + +using ::testing::_; +using ::testing::Eq; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Not; +using ::testing::Optional; +using ::testing::Return; +using ::testing::SetArgPointee; + +namespace webrtc { + +namespace { +static const size_t kMaxNumSamples = 48 * 10 * 2; // 10 ms @ 48 kHz stereo. +static const size_t kMockReturnEncodedBytes = 17; +static const int kCngPayloadType = 18; +} // namespace + +class AudioEncoderCngTest : public ::testing::Test { + protected: + AudioEncoderCngTest() + : mock_encoder_owner_(new MockAudioEncoder), + mock_encoder_(mock_encoder_owner_.get()), + mock_vad_(new MockVad), + timestamp_(4711), + num_audio_samples_10ms_(0), + sample_rate_hz_(8000) { + memset(audio_, 0, kMaxNumSamples * 2); + EXPECT_CALL(*mock_encoder_, NumChannels()).WillRepeatedly(Return(1)); + } + + AudioEncoderCngTest(const AudioEncoderCngTest&) = delete; + AudioEncoderCngTest& operator=(const AudioEncoderCngTest&) = delete; + + void TearDown() override { + EXPECT_CALL(*mock_vad_, Die()).Times(1); + cng_.reset(); + } + + AudioEncoderCngConfig MakeCngConfig() { + AudioEncoderCngConfig config; + config.speech_encoder = std::move(mock_encoder_owner_); + EXPECT_TRUE(config.speech_encoder); + + // Let the AudioEncoderCng object use a MockVad instead of its internally + // created Vad object. + config.vad = mock_vad_; + config.payload_type = kCngPayloadType; + + return config; + } + + void CreateCng(AudioEncoderCngConfig&& config) { + num_audio_samples_10ms_ = static_cast(10 * sample_rate_hz_ / 1000); + ASSERT_LE(num_audio_samples_10ms_, kMaxNumSamples); + if (config.speech_encoder) { + EXPECT_CALL(*mock_encoder_, SampleRateHz()) + .WillRepeatedly(Return(sample_rate_hz_)); + // Max10MsFramesInAPacket() is just used to verify that the SID frame + // period is not too small. The return value does not matter that much, + // as long as it is smaller than 10. + EXPECT_CALL(*mock_encoder_, Max10MsFramesInAPacket()) + .WillOnce(Return(1u)); + } + cng_ = CreateComfortNoiseEncoder(std::move(config)); + } + + void Encode() { + ASSERT_TRUE(cng_) << "Must call CreateCng() first."; + encoded_info_ = cng_->Encode( + timestamp_, + rtc::ArrayView(audio_, num_audio_samples_10ms_), + &encoded_); + timestamp_ += static_cast(num_audio_samples_10ms_); + } + + // Expect `num_calls` calls to the encoder, all successful. The last call + // claims to have encoded `kMockReturnEncodedBytes` bytes, and all the + // preceding ones 0 bytes. + void ExpectEncodeCalls(size_t num_calls) { + InSequence s; + AudioEncoder::EncodedInfo info; + for (size_t j = 0; j < num_calls - 1; ++j) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)).WillOnce(Return(info)); + } + info.encoded_bytes = kMockReturnEncodedBytes; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce( + Invoke(MockAudioEncoder::FakeEncoding(kMockReturnEncodedBytes))); + } + + // Verifies that the cng_ object waits until it has collected + // `blocks_per_frame` blocks of audio, and then dispatches all of them to + // the underlying codec (speech or cng). + void CheckBlockGrouping(size_t blocks_per_frame, bool active_speech) { + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(blocks_per_frame)); + auto config = MakeCngConfig(); + const int num_cng_coefficients = config.num_cng_coefficients; + CreateCng(std::move(config)); + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillRepeatedly(Return(active_speech ? Vad::kActive : Vad::kPassive)); + + // Don't expect any calls to the encoder yet. + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)).Times(0); + for (size_t i = 0; i < blocks_per_frame - 1; ++i) { + Encode(); + EXPECT_EQ(0u, encoded_info_.encoded_bytes); + } + if (active_speech) + ExpectEncodeCalls(blocks_per_frame); + Encode(); + if (active_speech) { + EXPECT_EQ(kMockReturnEncodedBytes, encoded_info_.encoded_bytes); + } else { + EXPECT_EQ(static_cast(num_cng_coefficients + 1), + encoded_info_.encoded_bytes); + } + } + + // Verifies that the audio is partitioned into larger blocks before calling + // the VAD. + void CheckVadInputSize(int input_frame_size_ms, + int expected_first_block_size_ms, + int expected_second_block_size_ms) { + const size_t blocks_per_frame = + static_cast(input_frame_size_ms / 10); + + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(blocks_per_frame)); + + // Expect nothing to happen before the last block is sent to cng_. + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)).Times(0); + for (size_t i = 0; i < blocks_per_frame - 1; ++i) { + Encode(); + } + + // Let the VAD decision be passive, since an active decision may lead to + // early termination of the decision loop. + InSequence s; + EXPECT_CALL( + *mock_vad_, + VoiceActivity(_, expected_first_block_size_ms * sample_rate_hz_ / 1000, + sample_rate_hz_)) + .WillOnce(Return(Vad::kPassive)); + if (expected_second_block_size_ms > 0) { + EXPECT_CALL(*mock_vad_, + VoiceActivity( + _, expected_second_block_size_ms * sample_rate_hz_ / 1000, + sample_rate_hz_)) + .WillOnce(Return(Vad::kPassive)); + } + + // With this call to Encode(), `mock_vad_` should be called according to the + // above expectations. + Encode(); + } + + // Tests a frame with both active and passive speech. Returns true if the + // decision was active speech, false if it was passive. + bool CheckMixedActivePassive(Vad::Activity first_type, + Vad::Activity second_type) { + // Set the speech encoder frame size to 60 ms, to ensure that the VAD will + // be called twice. + const size_t blocks_per_frame = 6; + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(blocks_per_frame)); + InSequence s; + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(first_type)); + if (first_type == Vad::kPassive) { + // Expect a second call to the VAD only if the first frame was passive. + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(second_type)); + } + encoded_info_.payload_type = 0; + for (size_t i = 0; i < blocks_per_frame; ++i) { + Encode(); + } + return encoded_info_.payload_type != kCngPayloadType; + } + + std::unique_ptr cng_; + std::unique_ptr mock_encoder_owner_; + MockAudioEncoder* mock_encoder_; + MockVad* mock_vad_; // Ownership is transferred to `cng_`. + uint32_t timestamp_; + int16_t audio_[kMaxNumSamples]; + size_t num_audio_samples_10ms_; + rtc::Buffer encoded_; + AudioEncoder::EncodedInfo encoded_info_; + int sample_rate_hz_; +}; + +TEST_F(AudioEncoderCngTest, CreateAndDestroy) { + CreateCng(MakeCngConfig()); +} + +TEST_F(AudioEncoderCngTest, CheckFrameSizePropagation) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillOnce(Return(17U)); + EXPECT_EQ(17U, cng_->Num10MsFramesInNextPacket()); +} + +TEST_F(AudioEncoderCngTest, CheckTargetAudioBitratePropagation) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, + OnReceivedUplinkBandwidth(4711, absl::optional())); + cng_->OnReceivedUplinkBandwidth(4711, absl::nullopt); +} + +TEST_F(AudioEncoderCngTest, CheckPacketLossFractionPropagation) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, OnReceivedUplinkPacketLossFraction(0.5)); + cng_->OnReceivedUplinkPacketLossFraction(0.5); +} + +TEST_F(AudioEncoderCngTest, CheckGetFrameLengthRangePropagation) { + CreateCng(MakeCngConfig()); + auto expected_range = + std::make_pair(TimeDelta::Millis(20), TimeDelta::Millis(20)); + EXPECT_CALL(*mock_encoder_, GetFrameLengthRange()) + .WillRepeatedly(Return(absl::make_optional(expected_range))); + EXPECT_THAT(cng_->GetFrameLengthRange(), Optional(Eq(expected_range))); +} + +TEST_F(AudioEncoderCngTest, EncodeCallsVad) { + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(1U)); + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(Vad::kPassive)); + Encode(); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects1BlockPassiveSpeech) { + CheckBlockGrouping(1, false); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects2BlocksPassiveSpeech) { + CheckBlockGrouping(2, false); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects3BlocksPassiveSpeech) { + CheckBlockGrouping(3, false); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects1BlockActiveSpeech) { + CheckBlockGrouping(1, true); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects2BlocksActiveSpeech) { + CheckBlockGrouping(2, true); +} + +TEST_F(AudioEncoderCngTest, EncodeCollects3BlocksActiveSpeech) { + CheckBlockGrouping(3, true); +} + +TEST_F(AudioEncoderCngTest, EncodePassive) { + const size_t kBlocksPerFrame = 3; + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(kBlocksPerFrame)); + auto config = MakeCngConfig(); + const auto sid_frame_interval_ms = config.sid_frame_interval_ms; + const auto num_cng_coefficients = config.num_cng_coefficients; + CreateCng(std::move(config)); + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillRepeatedly(Return(Vad::kPassive)); + // Expect no calls at all to the speech encoder mock. + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)).Times(0); + uint32_t expected_timestamp = timestamp_; + for (size_t i = 0; i < 100; ++i) { + Encode(); + // Check if it was time to call the cng encoder. This is done once every + // `kBlocksPerFrame` calls. + if ((i + 1) % kBlocksPerFrame == 0) { + // Now check if a SID interval has elapsed. + if ((i % (sid_frame_interval_ms / 10)) < kBlocksPerFrame) { + // If so, verify that we got a CNG encoding. + EXPECT_EQ(kCngPayloadType, encoded_info_.payload_type); + EXPECT_FALSE(encoded_info_.speech); + EXPECT_EQ(static_cast(num_cng_coefficients) + 1, + encoded_info_.encoded_bytes); + EXPECT_EQ(expected_timestamp, encoded_info_.encoded_timestamp); + } + expected_timestamp += rtc::checked_cast( + kBlocksPerFrame * num_audio_samples_10ms_); + } else { + // Otherwise, expect no output. + EXPECT_EQ(0u, encoded_info_.encoded_bytes); + } + } +} + +// Verifies that the correct action is taken for frames with both active and +// passive speech. +TEST_F(AudioEncoderCngTest, MixedActivePassive) { + CreateCng(MakeCngConfig()); + + // All of the frame is active speech. + ExpectEncodeCalls(6); + EXPECT_TRUE(CheckMixedActivePassive(Vad::kActive, Vad::kActive)); + EXPECT_TRUE(encoded_info_.speech); + + // First half of the frame is active speech. + ExpectEncodeCalls(6); + EXPECT_TRUE(CheckMixedActivePassive(Vad::kActive, Vad::kPassive)); + EXPECT_TRUE(encoded_info_.speech); + + // Second half of the frame is active speech. + ExpectEncodeCalls(6); + EXPECT_TRUE(CheckMixedActivePassive(Vad::kPassive, Vad::kActive)); + EXPECT_TRUE(encoded_info_.speech); + + // All of the frame is passive speech. Expect no calls to `mock_encoder_`. + EXPECT_FALSE(CheckMixedActivePassive(Vad::kPassive, Vad::kPassive)); + EXPECT_FALSE(encoded_info_.speech); +} + +// These tests verify that the audio is partitioned into larger blocks before +// calling the VAD. +// The parameters for CheckVadInputSize are: +// CheckVadInputSize(frame_size, expected_first_block_size, +// expected_second_block_size); +TEST_F(AudioEncoderCngTest, VadInputSize10Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(10, 10, 0); +} +TEST_F(AudioEncoderCngTest, VadInputSize20Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(20, 20, 0); +} +TEST_F(AudioEncoderCngTest, VadInputSize30Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(30, 30, 0); +} +TEST_F(AudioEncoderCngTest, VadInputSize40Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(40, 20, 20); +} +TEST_F(AudioEncoderCngTest, VadInputSize50Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(50, 30, 20); +} +TEST_F(AudioEncoderCngTest, VadInputSize60Ms) { + CreateCng(MakeCngConfig()); + CheckVadInputSize(60, 30, 30); +} + +// Verifies that the correct payload type is set when CNG is encoded. +TEST_F(AudioEncoderCngTest, VerifyCngPayloadType) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)).Times(0); + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()).WillOnce(Return(1U)); + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(Vad::kPassive)); + encoded_info_.payload_type = 0; + Encode(); + EXPECT_EQ(kCngPayloadType, encoded_info_.payload_type); +} + +// Verifies that a SID frame is encoded immediately as the signal changes from +// active speech to passive. +TEST_F(AudioEncoderCngTest, VerifySidFrameAfterSpeech) { + auto config = MakeCngConfig(); + const auto num_cng_coefficients = config.num_cng_coefficients; + CreateCng(std::move(config)); + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(1U)); + // Start with encoding noise. + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .Times(2) + .WillRepeatedly(Return(Vad::kPassive)); + Encode(); + EXPECT_EQ(kCngPayloadType, encoded_info_.payload_type); + EXPECT_EQ(static_cast(num_cng_coefficients) + 1, + encoded_info_.encoded_bytes); + // Encode again, and make sure we got no frame at all (since the SID frame + // period is 100 ms by default). + Encode(); + EXPECT_EQ(0u, encoded_info_.encoded_bytes); + + // Now encode active speech. + encoded_info_.payload_type = 0; + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(Vad::kActive)); + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce( + Invoke(MockAudioEncoder::FakeEncoding(kMockReturnEncodedBytes))); + Encode(); + EXPECT_EQ(kMockReturnEncodedBytes, encoded_info_.encoded_bytes); + + // Go back to noise again, and verify that a SID frame is emitted. + EXPECT_CALL(*mock_vad_, VoiceActivity(_, _, _)) + .WillOnce(Return(Vad::kPassive)); + Encode(); + EXPECT_EQ(kCngPayloadType, encoded_info_.payload_type); + EXPECT_EQ(static_cast(num_cng_coefficients) + 1, + encoded_info_.encoded_bytes); +} + +// Resetting the CNG should reset both the VAD and the encoder. +TEST_F(AudioEncoderCngTest, Reset) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, Reset()).Times(1); + EXPECT_CALL(*mock_vad_, Reset()).Times(1); + cng_->Reset(); +} + +#if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// This test fixture tests various error conditions that makes the +// AudioEncoderCng die via CHECKs. +class AudioEncoderCngDeathTest : public AudioEncoderCngTest { + protected: + AudioEncoderCngDeathTest() : AudioEncoderCngTest() { + EXPECT_CALL(*mock_vad_, Die()).Times(1); + delete mock_vad_; + mock_vad_ = nullptr; + } + + // Override AudioEncoderCngTest::TearDown, since that one expects a call to + // the destructor of `mock_vad_`. In this case, that object is already + // deleted. + void TearDown() override { cng_.reset(); } + + AudioEncoderCngConfig MakeCngConfig() { + // Don't provide a Vad mock object, since it would leak when the test dies. + auto config = AudioEncoderCngTest::MakeCngConfig(); + config.vad = nullptr; + return config; + } + + void TryWrongNumCoefficients(int num) { + RTC_EXPECT_DEATH( + [&] { + auto config = MakeCngConfig(); + config.num_cng_coefficients = num; + CreateCng(std::move(config)); + }(), + "Invalid configuration"); + } +}; + +TEST_F(AudioEncoderCngDeathTest, WrongFrameSize) { + CreateCng(MakeCngConfig()); + num_audio_samples_10ms_ *= 2; // 20 ms frame. + RTC_EXPECT_DEATH(Encode(), ""); + num_audio_samples_10ms_ = 0; // Zero samples. + RTC_EXPECT_DEATH(Encode(), ""); +} + +TEST_F(AudioEncoderCngDeathTest, WrongNumCoefficientsA) { + TryWrongNumCoefficients(-1); +} + +TEST_F(AudioEncoderCngDeathTest, WrongNumCoefficientsB) { + TryWrongNumCoefficients(0); +} + +TEST_F(AudioEncoderCngDeathTest, WrongNumCoefficientsC) { + TryWrongNumCoefficients(13); +} + +TEST_F(AudioEncoderCngDeathTest, NullSpeechEncoder) { + auto config = MakeCngConfig(); + config.speech_encoder = nullptr; + RTC_EXPECT_DEATH(CreateCng(std::move(config)), ""); +} + +TEST_F(AudioEncoderCngDeathTest, StereoEncoder) { + EXPECT_CALL(*mock_encoder_, NumChannels()).WillRepeatedly(Return(2)); + RTC_EXPECT_DEATH(CreateCng(MakeCngConfig()), "Invalid configuration"); +} + +TEST_F(AudioEncoderCngDeathTest, StereoConfig) { + RTC_EXPECT_DEATH( + [&] { + auto config = MakeCngConfig(); + config.num_channels = 2; + CreateCng(std::move(config)); + }(), + "Invalid configuration"); +} + +TEST_F(AudioEncoderCngDeathTest, EncoderFrameSizeTooLarge) { + CreateCng(MakeCngConfig()); + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillRepeatedly(Return(7U)); + for (int i = 0; i < 6; ++i) + Encode(); + RTC_EXPECT_DEATH( + Encode(), "Frame size cannot be larger than 60 ms when using VAD/CNG."); +} + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/cng/cng_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/cng/cng_unittest.cc new file mode 100644 index 0000000000..0e6ab79394 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/cng/cng_unittest.cc @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include +#include + +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +enum { + kSidShortIntervalUpdate = 1, + kSidNormalIntervalUpdate = 100, + kSidLongIntervalUpdate = 10000 +}; + +enum : size_t { + kCNGNumParamsLow = 0, + kCNGNumParamsNormal = 8, + kCNGNumParamsHigh = WEBRTC_CNG_MAX_LPC_ORDER, + kCNGNumParamsTooHigh = WEBRTC_CNG_MAX_LPC_ORDER + 1 +}; + +enum { kNoSid, kForceSid }; + +class CngTest : public ::testing::Test { + protected: + virtual void SetUp(); + + void TestCngEncode(int sample_rate_hz, int quality); + + int16_t speech_data_[640]; // Max size of CNG internal buffers. +}; + +class CngDeathTest : public CngTest {}; + +void CngTest::SetUp() { + FILE* input_file; + const std::string file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + input_file = fopen(file_name.c_str(), "rb"); + ASSERT_TRUE(input_file != NULL); + ASSERT_EQ(640, static_cast( + fread(speech_data_, sizeof(int16_t), 640, input_file))); + fclose(input_file); + input_file = NULL; +} + +void CngTest::TestCngEncode(int sample_rate_hz, int quality) { + const size_t num_samples_10ms = rtc::CheckedDivExact(sample_rate_hz, 100); + rtc::Buffer sid_data; + + ComfortNoiseEncoder cng_encoder(sample_rate_hz, kSidNormalIntervalUpdate, + quality); + EXPECT_EQ(0U, cng_encoder.Encode(rtc::ArrayView( + speech_data_, num_samples_10ms), + kNoSid, &sid_data)); + EXPECT_EQ(static_cast(quality + 1), + cng_encoder.Encode( + rtc::ArrayView(speech_data_, num_samples_10ms), + kForceSid, &sid_data)); +} + +#if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +// Create CNG encoder, init with faulty values, free CNG encoder. +TEST_F(CngDeathTest, CngInitFail) { + // Call with too few parameters. + EXPECT_DEATH( + { + ComfortNoiseEncoder(8000, kSidNormalIntervalUpdate, kCNGNumParamsLow); + }, + ""); + // Call with too many parameters. + EXPECT_DEATH( + { + ComfortNoiseEncoder(8000, kSidNormalIntervalUpdate, + kCNGNumParamsTooHigh); + }, + ""); +} + +// Encode Cng with too long input vector. +TEST_F(CngDeathTest, CngEncodeTooLong) { + rtc::Buffer sid_data; + + // Create encoder. + ComfortNoiseEncoder cng_encoder(8000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + // Run encoder with too much data. + EXPECT_DEATH( + cng_encoder.Encode(rtc::ArrayView(speech_data_, 641), + kNoSid, &sid_data), + ""); +} +#endif // GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +TEST_F(CngTest, CngEncode8000) { + TestCngEncode(8000, kCNGNumParamsNormal); +} + +TEST_F(CngTest, CngEncode16000) { + TestCngEncode(16000, kCNGNumParamsNormal); +} + +TEST_F(CngTest, CngEncode32000) { + TestCngEncode(32000, kCNGNumParamsHigh); +} + +TEST_F(CngTest, CngEncode48000) { + TestCngEncode(48000, kCNGNumParamsNormal); +} + +TEST_F(CngTest, CngEncode64000) { + TestCngEncode(64000, kCNGNumParamsNormal); +} + +// Update SID parameters, for both 9 and 16 parameters. +TEST_F(CngTest, CngUpdateSid) { + rtc::Buffer sid_data; + + // Create and initialize encoder and decoder. + ComfortNoiseEncoder cng_encoder(16000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + + // Run normal Encode and UpdateSid. + EXPECT_EQ(kCNGNumParamsNormal + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kForceSid, &sid_data)); + cng_decoder.UpdateSid(sid_data); + + // Reinit with new length. + cng_encoder.Reset(16000, kSidNormalIntervalUpdate, kCNGNumParamsHigh); + cng_decoder.Reset(); + + // Expect 0 because of unstable parameters after switching length. + EXPECT_EQ(0U, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kForceSid, &sid_data)); + EXPECT_EQ( + kCNGNumParamsHigh + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_ + 160, 160), + kForceSid, &sid_data)); + cng_decoder.UpdateSid( + rtc::ArrayView(sid_data.data(), kCNGNumParamsNormal + 1)); +} + +// Update SID parameters, with wrong parameters or without calling decode. +TEST_F(CngTest, CngUpdateSidErroneous) { + rtc::Buffer sid_data; + + // Encode. + ComfortNoiseEncoder cng_encoder(16000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + EXPECT_EQ(kCNGNumParamsNormal + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kForceSid, &sid_data)); + + // First run with valid parameters, then with too many CNG parameters. + // The function will operate correctly by only reading the maximum number of + // parameters, skipping the extra. + EXPECT_EQ(kCNGNumParamsNormal + 1, sid_data.size()); + cng_decoder.UpdateSid(sid_data); + + // Make sure the input buffer is large enough. Since Encode() appends data, we + // need to set the size manually only afterwards, or the buffer will be bigger + // than anticipated. + sid_data.SetSize(kCNGNumParamsTooHigh + 1); + cng_decoder.UpdateSid(sid_data); +} + +// Test to generate cng data, by forcing SID. Both normal and faulty condition. +TEST_F(CngTest, CngGenerate) { + rtc::Buffer sid_data; + int16_t out_data[640]; + + // Create and initialize encoder and decoder. + ComfortNoiseEncoder cng_encoder(16000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + + // Normal Encode. + EXPECT_EQ(kCNGNumParamsNormal + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kForceSid, &sid_data)); + + // Normal UpdateSid. + cng_decoder.UpdateSid(sid_data); + + // Two normal Generate, one with new_period. + EXPECT_TRUE(cng_decoder.Generate(rtc::ArrayView(out_data, 640), 1)); + EXPECT_TRUE(cng_decoder.Generate(rtc::ArrayView(out_data, 640), 0)); + + // Call Genereate with too much data. + EXPECT_FALSE(cng_decoder.Generate(rtc::ArrayView(out_data, 641), 0)); +} + +// Test automatic SID. +TEST_F(CngTest, CngAutoSid) { + rtc::Buffer sid_data; + + // Create and initialize encoder and decoder. + ComfortNoiseEncoder cng_encoder(16000, kSidNormalIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + + // Normal Encode, 100 msec, where no SID data should be generated. + for (int i = 0; i < 10; i++) { + EXPECT_EQ( + 0U, cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kNoSid, &sid_data)); + } + + // We have reached 100 msec, and SID data should be generated. + EXPECT_EQ(kCNGNumParamsNormal + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kNoSid, &sid_data)); +} + +// Test automatic SID, with very short interval. +TEST_F(CngTest, CngAutoSidShort) { + rtc::Buffer sid_data; + + // Create and initialize encoder and decoder. + ComfortNoiseEncoder cng_encoder(16000, kSidShortIntervalUpdate, + kCNGNumParamsNormal); + ComfortNoiseDecoder cng_decoder; + + // First call will never generate SID, unless forced to. + EXPECT_EQ(0U, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kNoSid, &sid_data)); + + // Normal Encode, 100 msec, SID data should be generated all the time. + for (int i = 0; i < 10; i++) { + EXPECT_EQ( + kCNGNumParamsNormal + 1, + cng_encoder.Encode(rtc::ArrayView(speech_data_, 160), + kNoSid, &sid_data)); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc b/third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc new file mode 100644 index 0000000000..48f1b8c296 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" + +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +namespace { + +const size_t kCngMaxOutsizeOrder = 640; + +// TODO(ossu): Rename the left-over WebRtcCng according to style guide. +void WebRtcCng_K2a16(int16_t* k, int useOrder, int16_t* a); + +const int32_t WebRtcCng_kDbov[94] = { + 1081109975, 858756178, 682134279, 541838517, 430397633, 341876992, + 271562548, 215709799, 171344384, 136103682, 108110997, 85875618, + 68213428, 54183852, 43039763, 34187699, 27156255, 21570980, + 17134438, 13610368, 10811100, 8587562, 6821343, 5418385, + 4303976, 3418770, 2715625, 2157098, 1713444, 1361037, + 1081110, 858756, 682134, 541839, 430398, 341877, + 271563, 215710, 171344, 136104, 108111, 85876, + 68213, 54184, 43040, 34188, 27156, 21571, + 17134, 13610, 10811, 8588, 6821, 5418, + 4304, 3419, 2716, 2157, 1713, 1361, + 1081, 859, 682, 542, 430, 342, + 272, 216, 171, 136, 108, 86, + 68, 54, 43, 34, 27, 22, + 17, 14, 11, 9, 7, 5, + 4, 3, 3, 2, 2, 1, + 1, 1, 1, 1}; + +const int16_t WebRtcCng_kCorrWindow[WEBRTC_CNG_MAX_LPC_ORDER] = { + 32702, 32636, 32570, 32505, 32439, 32374, + 32309, 32244, 32179, 32114, 32049, 31985}; + +} // namespace + +ComfortNoiseDecoder::ComfortNoiseDecoder() { + /* Needed to get the right function pointers in SPLIB. */ + Reset(); +} + +void ComfortNoiseDecoder::Reset() { + dec_seed_ = 7777; /* For debugging only. */ + dec_target_energy_ = 0; + dec_used_energy_ = 0; + for (auto& c : dec_target_reflCoefs_) + c = 0; + for (auto& c : dec_used_reflCoefs_) + c = 0; + for (auto& c : dec_filtstate_) + c = 0; + for (auto& c : dec_filtstateLow_) + c = 0; + dec_order_ = 5; + dec_target_scale_factor_ = 0; + dec_used_scale_factor_ = 0; +} + +void ComfortNoiseDecoder::UpdateSid(rtc::ArrayView sid) { + int16_t refCs[WEBRTC_CNG_MAX_LPC_ORDER]; + int32_t targetEnergy; + size_t length = sid.size(); + /* Throw away reflection coefficients of higher order than we can handle. */ + if (length > (WEBRTC_CNG_MAX_LPC_ORDER + 1)) + length = WEBRTC_CNG_MAX_LPC_ORDER + 1; + + dec_order_ = static_cast(length - 1); + + uint8_t sid0 = std::min(sid[0], 93); + targetEnergy = WebRtcCng_kDbov[sid0]; + /* Take down target energy to 75%. */ + targetEnergy = targetEnergy >> 1; + targetEnergy += targetEnergy >> 2; + + dec_target_energy_ = targetEnergy; + + /* Reconstruct coeffs with tweak for WebRtc implementation of RFC3389. */ + if (dec_order_ == WEBRTC_CNG_MAX_LPC_ORDER) { + for (size_t i = 0; i < (dec_order_); i++) { + refCs[i] = sid[i + 1] << 8; /* Q7 to Q15*/ + dec_target_reflCoefs_[i] = refCs[i]; + } + } else { + for (size_t i = 0; i < (dec_order_); i++) { + refCs[i] = (sid[i + 1] - 127) * (1 << 8); /* Q7 to Q15. */ + dec_target_reflCoefs_[i] = refCs[i]; + } + } + + for (size_t i = (dec_order_); i < WEBRTC_CNG_MAX_LPC_ORDER; i++) { + refCs[i] = 0; + dec_target_reflCoefs_[i] = refCs[i]; + } +} + +bool ComfortNoiseDecoder::Generate(rtc::ArrayView out_data, + bool new_period) { + int16_t excitation[kCngMaxOutsizeOrder]; + int16_t low[kCngMaxOutsizeOrder]; + int16_t lpPoly[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t ReflBetaStd = 26214; /* 0.8 in q15. */ + int16_t ReflBetaCompStd = 6553; /* 0.2 in q15. */ + int16_t ReflBetaNewP = 19661; /* 0.6 in q15. */ + int16_t ReflBetaCompNewP = 13107; /* 0.4 in q15. */ + int16_t Beta, BetaC; /* These are in Q15. */ + int32_t targetEnergy; + int16_t En; + int16_t temp16; + const size_t num_samples = out_data.size(); + + if (num_samples > kCngMaxOutsizeOrder) { + return false; + } + + if (new_period) { + dec_used_scale_factor_ = dec_target_scale_factor_; + Beta = ReflBetaNewP; + BetaC = ReflBetaCompNewP; + } else { + Beta = ReflBetaStd; + BetaC = ReflBetaCompStd; + } + + /* Calculate new scale factor in Q13 */ + dec_used_scale_factor_ = rtc::checked_cast( + WEBRTC_SPL_MUL_16_16_RSFT(dec_used_scale_factor_, Beta >> 2, 13) + + WEBRTC_SPL_MUL_16_16_RSFT(dec_target_scale_factor_, BetaC >> 2, 13)); + + dec_used_energy_ = dec_used_energy_ >> 1; + dec_used_energy_ += dec_target_energy_ >> 1; + + /* Do the same for the reflection coeffs, albeit in Q15. */ + for (size_t i = 0; i < WEBRTC_CNG_MAX_LPC_ORDER; i++) { + dec_used_reflCoefs_[i] = + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(dec_used_reflCoefs_[i], Beta, 15); + dec_used_reflCoefs_[i] += + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(dec_target_reflCoefs_[i], BetaC, 15); + } + + /* Compute the polynomial coefficients. */ + WebRtcCng_K2a16(dec_used_reflCoefs_, WEBRTC_CNG_MAX_LPC_ORDER, lpPoly); + + targetEnergy = dec_used_energy_; + + /* Calculate scaling factor based on filter energy. */ + En = 8192; /* 1.0 in Q13. */ + for (size_t i = 0; i < (WEBRTC_CNG_MAX_LPC_ORDER); i++) { + /* Floating point value for reference. + E *= 1.0 - (dec_used_reflCoefs_[i] / 32768.0) * + (dec_used_reflCoefs_[i] / 32768.0); + */ + + /* Same in fixed point. */ + /* K(i).^2 in Q15. */ + temp16 = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(dec_used_reflCoefs_[i], + dec_used_reflCoefs_[i], 15); + /* 1 - K(i).^2 in Q15. */ + temp16 = 0x7fff - temp16; + En = (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(En, temp16, 15); + } + + /* float scaling= sqrt(E * dec_target_energy_ / (1 << 24)); */ + + /* Calculate sqrt(En * target_energy / excitation energy) */ + targetEnergy = WebRtcSpl_Sqrt(dec_used_energy_); + + En = (int16_t)WebRtcSpl_Sqrt(En) << 6; + En = (En * 3) >> 1; /* 1.5 estimates sqrt(2). */ + dec_used_scale_factor_ = (int16_t)((En * targetEnergy) >> 12); + + /* Generate excitation. */ + /* Excitation energy per sample is 2.^24 - Q13 N(0,1). */ + for (size_t i = 0; i < num_samples; i++) { + excitation[i] = WebRtcSpl_RandN(&dec_seed_) >> 1; + } + + /* Scale to correct energy. */ + WebRtcSpl_ScaleVector(excitation, excitation, dec_used_scale_factor_, + num_samples, 13); + + /* `lpPoly` - Coefficients in Q12. + * `excitation` - Speech samples. + * `nst->dec_filtstate` - State preservation. + * `out_data` - Filtered speech samples. */ + WebRtcSpl_FilterAR(lpPoly, WEBRTC_CNG_MAX_LPC_ORDER + 1, excitation, + num_samples, dec_filtstate_, WEBRTC_CNG_MAX_LPC_ORDER, + dec_filtstateLow_, WEBRTC_CNG_MAX_LPC_ORDER, + out_data.data(), low, num_samples); + + return true; +} + +ComfortNoiseEncoder::ComfortNoiseEncoder(int fs, int interval, int quality) + : enc_nrOfCoefs_(quality), + enc_sampfreq_(fs), + enc_interval_(interval), + enc_msSinceSid_(0), + enc_Energy_(0), + enc_reflCoefs_{0}, + enc_corrVector_{0}, + enc_seed_(7777) /* For debugging only. */ { + RTC_CHECK_GT(quality, 0); + RTC_CHECK_LE(quality, WEBRTC_CNG_MAX_LPC_ORDER); +} + +void ComfortNoiseEncoder::Reset(int fs, int interval, int quality) { + RTC_CHECK_GT(quality, 0); + RTC_CHECK_LE(quality, WEBRTC_CNG_MAX_LPC_ORDER); + enc_nrOfCoefs_ = quality; + enc_sampfreq_ = fs; + enc_interval_ = interval; + enc_msSinceSid_ = 0; + enc_Energy_ = 0; + for (auto& c : enc_reflCoefs_) + c = 0; + for (auto& c : enc_corrVector_) + c = 0; + enc_seed_ = 7777; /* For debugging only. */ +} + +size_t ComfortNoiseEncoder::Encode(rtc::ArrayView speech, + bool force_sid, + rtc::Buffer* output) { + int16_t arCoefs[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int32_t corrVector[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t refCs[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t hanningW[kCngMaxOutsizeOrder]; + int16_t ReflBeta = 19661; /* 0.6 in q15. */ + int16_t ReflBetaComp = 13107; /* 0.4 in q15. */ + int32_t outEnergy; + int outShifts; + size_t i; + int stab; + int acorrScale; + size_t index; + size_t ind, factor; + int32_t* bptr; + int32_t blo, bhi; + int16_t negate; + const int16_t* aptr; + int16_t speechBuf[kCngMaxOutsizeOrder]; + + const size_t num_samples = speech.size(); + RTC_CHECK_LE(num_samples, kCngMaxOutsizeOrder); + + for (i = 0; i < num_samples; i++) { + speechBuf[i] = speech[i]; + } + + factor = num_samples; + + /* Calculate energy and a coefficients. */ + outEnergy = WebRtcSpl_Energy(speechBuf, num_samples, &outShifts); + while (outShifts > 0) { + /* We can only do 5 shifts without destroying accuracy in + * division factor. */ + if (outShifts > 5) { + outEnergy <<= (outShifts - 5); + outShifts = 5; + } else { + factor /= 2; + outShifts--; + } + } + outEnergy = WebRtcSpl_DivW32W16(outEnergy, (int16_t)factor); + + if (outEnergy > 1) { + /* Create Hanning Window. */ + WebRtcSpl_GetHanningWindow(hanningW, num_samples / 2); + for (i = 0; i < (num_samples / 2); i++) + hanningW[num_samples - i - 1] = hanningW[i]; + + WebRtcSpl_ElementwiseVectorMult(speechBuf, hanningW, speechBuf, num_samples, + 14); + + WebRtcSpl_AutoCorrelation(speechBuf, num_samples, enc_nrOfCoefs_, + corrVector, &acorrScale); + + if (*corrVector == 0) + *corrVector = WEBRTC_SPL_WORD16_MAX; + + /* Adds the bandwidth expansion. */ + aptr = WebRtcCng_kCorrWindow; + bptr = corrVector; + + /* (zzz) lpc16_1 = 17+1+820+2+2 = 842 (ordo2=700). */ + for (ind = 0; ind < enc_nrOfCoefs_; ind++) { + /* The below code multiplies the 16 b corrWindow values (Q15) with + * the 32 b corrvector (Q0) and shifts the result down 15 steps. */ + negate = *bptr < 0; + if (negate) + *bptr = -*bptr; + + blo = (int32_t)*aptr * (*bptr & 0xffff); + bhi = ((blo >> 16) & 0xffff) + + ((int32_t)(*aptr++) * ((*bptr >> 16) & 0xffff)); + blo = (blo & 0xffff) | ((bhi & 0xffff) << 16); + + *bptr = (((bhi >> 16) & 0x7fff) << 17) | ((uint32_t)blo >> 15); + if (negate) + *bptr = -*bptr; + bptr++; + } + /* End of bandwidth expansion. */ + + stab = WebRtcSpl_LevinsonDurbin(corrVector, arCoefs, refCs, enc_nrOfCoefs_); + + if (!stab) { + /* Disregard from this frame */ + return 0; + } + + } else { + for (i = 0; i < enc_nrOfCoefs_; i++) + refCs[i] = 0; + } + + if (force_sid) { + /* Read instantaneous values instead of averaged. */ + for (i = 0; i < enc_nrOfCoefs_; i++) + enc_reflCoefs_[i] = refCs[i]; + enc_Energy_ = outEnergy; + } else { + /* Average history with new values. */ + for (i = 0; i < enc_nrOfCoefs_; i++) { + enc_reflCoefs_[i] = + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(enc_reflCoefs_[i], ReflBeta, 15); + enc_reflCoefs_[i] += + (int16_t)WEBRTC_SPL_MUL_16_16_RSFT(refCs[i], ReflBetaComp, 15); + } + enc_Energy_ = (outEnergy >> 2) + (enc_Energy_ >> 1) + (enc_Energy_ >> 2); + } + + if (enc_Energy_ < 1) { + enc_Energy_ = 1; + } + + if ((enc_msSinceSid_ > (enc_interval_ - 1)) || force_sid) { + /* Search for best dbov value. */ + index = 0; + for (i = 1; i < 93; i++) { + /* Always round downwards. */ + if ((enc_Energy_ - WebRtcCng_kDbov[i]) > 0) { + index = i; + break; + } + } + if ((i == 93) && (index == 0)) + index = 94; + + const size_t output_coefs = enc_nrOfCoefs_ + 1; + output->AppendData(output_coefs, [&](rtc::ArrayView output) { + output[0] = (uint8_t)index; + + /* Quantize coefficients with tweak for WebRtc implementation of + * RFC3389. */ + if (enc_nrOfCoefs_ == WEBRTC_CNG_MAX_LPC_ORDER) { + for (i = 0; i < enc_nrOfCoefs_; i++) { + /* Q15 to Q7 with rounding. */ + output[i + 1] = ((enc_reflCoefs_[i] + 128) >> 8); + } + } else { + for (i = 0; i < enc_nrOfCoefs_; i++) { + /* Q15 to Q7 with rounding. */ + output[i + 1] = (127 + ((enc_reflCoefs_[i] + 128) >> 8)); + } + } + + return output_coefs; + }); + + enc_msSinceSid_ = + static_cast((1000 * num_samples) / enc_sampfreq_); + return output_coefs; + } else { + enc_msSinceSid_ += + static_cast((1000 * num_samples) / enc_sampfreq_); + return 0; + } +} + +namespace { +/* Values in `k` are Q15, and `a` Q12. */ +void WebRtcCng_K2a16(int16_t* k, int useOrder, int16_t* a) { + int16_t any[WEBRTC_SPL_MAX_LPC_ORDER + 1]; + int16_t* aptr; + int16_t* aptr2; + int16_t* anyptr; + const int16_t* kptr; + int m, i; + + kptr = k; + *a = 4096; /* i.e., (Word16_MAX >> 3) + 1 */ + *any = *a; + a[1] = (*k + 4) >> 3; + for (m = 1; m < useOrder; m++) { + kptr++; + aptr = a; + aptr++; + aptr2 = &a[m]; + anyptr = any; + anyptr++; + + any[m + 1] = (*kptr + 4) >> 3; + for (i = 0; i < m; i++) { + *anyptr++ = + (*aptr++) + + (int16_t)((((int32_t)(*aptr2--) * (int32_t)*kptr) + 16384) >> 15); + } + + aptr = a; + anyptr = any; + for (i = 0; i < (m + 2); i++) { + *aptr++ = *anyptr++; + } + } +} + +} // namespace + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.h b/third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.h new file mode 100644 index 0000000000..7afd243f81 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_CNG_WEBRTC_CNG_H_ +#define MODULES_AUDIO_CODING_CODECS_CNG_WEBRTC_CNG_H_ + +#include + +#include + +#include "api/array_view.h" +#include "rtc_base/buffer.h" + +#define WEBRTC_CNG_MAX_LPC_ORDER 12 + +namespace webrtc { + +class ComfortNoiseDecoder { + public: + ComfortNoiseDecoder(); + ~ComfortNoiseDecoder() = default; + + ComfortNoiseDecoder(const ComfortNoiseDecoder&) = delete; + ComfortNoiseDecoder& operator=(const ComfortNoiseDecoder&) = delete; + + void Reset(); + + // Updates the CN state when a new SID packet arrives. + // `sid` is a view of the SID packet without the headers. + void UpdateSid(rtc::ArrayView sid); + + // Generates comfort noise. + // `out_data` will be filled with samples - its size determines the number of + // samples generated. When `new_period` is true, CNG history will be reset + // before any audio is generated. Returns `false` if outData is too large - + // currently 640 bytes (equalling 10ms at 64kHz). + // TODO(ossu): Specify better limits for the size of out_data. Either let it + // be unbounded or limit to 10ms in the current sample rate. + bool Generate(rtc::ArrayView out_data, bool new_period); + + private: + uint32_t dec_seed_; + int32_t dec_target_energy_; + int32_t dec_used_energy_; + int16_t dec_target_reflCoefs_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t dec_used_reflCoefs_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t dec_filtstate_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int16_t dec_filtstateLow_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + uint16_t dec_order_; + int16_t dec_target_scale_factor_; /* Q29 */ + int16_t dec_used_scale_factor_; /* Q29 */ +}; + +class ComfortNoiseEncoder { + public: + // Creates a comfort noise encoder. + // `fs` selects sample rate: 8000 for narrowband or 16000 for wideband. + // `interval` sets the interval at which to generate SID data (in ms). + // `quality` selects the number of refl. coeffs. Maximum allowed is 12. + ComfortNoiseEncoder(int fs, int interval, int quality); + ~ComfortNoiseEncoder() = default; + + ComfortNoiseEncoder(const ComfortNoiseEncoder&) = delete; + ComfortNoiseEncoder& operator=(const ComfortNoiseEncoder&) = delete; + + // Resets the comfort noise encoder to its initial state. + // Parameters are set as during construction. + void Reset(int fs, int interval, int quality); + + // Analyzes background noise from `speech` and appends coefficients to + // `output`. Returns the number of coefficients generated. If `force_sid` is + // true, a SID frame is forced and the internal sid interval counter is reset. + // Will fail if the input size is too large (> 640 samples, see + // ComfortNoiseDecoder::Generate). + size_t Encode(rtc::ArrayView speech, + bool force_sid, + rtc::Buffer* output); + + private: + size_t enc_nrOfCoefs_; + int enc_sampfreq_; + int16_t enc_interval_; + int16_t enc_msSinceSid_; + int32_t enc_Energy_; + int16_t enc_reflCoefs_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + int32_t enc_corrVector_[WEBRTC_CNG_MAX_LPC_ORDER + 1]; + uint32_t enc_seed_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_CNG_WEBRTC_CNG_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc b/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc new file mode 100644 index 0000000000..46ac671b30 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h" + +#include + +#include "modules/audio_coding/codecs/g711/g711_interface.h" +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" + +namespace webrtc { + +void AudioDecoderPcmU::Reset() {} + +std::vector AudioDecoderPcmU::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, 8 * num_channels_, 8); +} + +int AudioDecoderPcmU::SampleRateHz() const { + return 8000; +} + +size_t AudioDecoderPcmU::Channels() const { + return num_channels_; +} + +int AudioDecoderPcmU::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(SampleRateHz(), sample_rate_hz); + // Adjust the encoded length down to ensure the same number of samples in each + // channel. + const size_t encoded_len_adjusted = + PacketDuration(encoded, encoded_len) * + Channels(); // 1 byte per sample per channel + int16_t temp_type = 1; // Default is speech. + size_t ret = + WebRtcG711_DecodeU(encoded, encoded_len_adjusted, decoded, &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return static_cast(ret); +} + +int AudioDecoderPcmU::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // One encoded byte per sample per channel. + return static_cast(encoded_len / Channels()); +} + +void AudioDecoderPcmA::Reset() {} + +std::vector AudioDecoderPcmA::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, 8 * num_channels_, 8); +} + +int AudioDecoderPcmA::SampleRateHz() const { + return 8000; +} + +size_t AudioDecoderPcmA::Channels() const { + return num_channels_; +} + +int AudioDecoderPcmA::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(SampleRateHz(), sample_rate_hz); + // Adjust the encoded length down to ensure the same number of samples in each + // channel. + const size_t encoded_len_adjusted = + PacketDuration(encoded, encoded_len) * + Channels(); // 1 byte per sample per channel + int16_t temp_type = 1; // Default is speech. + size_t ret = + WebRtcG711_DecodeA(encoded, encoded_len_adjusted, decoded, &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return static_cast(ret); +} + +int AudioDecoderPcmA::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // One encoded byte per sample per channel. + return static_cast(encoded_len / Channels()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h b/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h new file mode 100644 index 0000000000..3fa42cba30 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G711_AUDIO_DECODER_PCM_H_ +#define MODULES_AUDIO_CODING_CODECS_G711_AUDIO_DECODER_PCM_H_ + +#include +#include + +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/buffer.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +class AudioDecoderPcmU final : public AudioDecoder { + public: + explicit AudioDecoderPcmU(size_t num_channels) : num_channels_(num_channels) { + RTC_DCHECK_GE(num_channels, 1); + } + + AudioDecoderPcmU(const AudioDecoderPcmU&) = delete; + AudioDecoderPcmU& operator=(const AudioDecoderPcmU&) = delete; + + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + const size_t num_channels_; +}; + +class AudioDecoderPcmA final : public AudioDecoder { + public: + explicit AudioDecoderPcmA(size_t num_channels) : num_channels_(num_channels) { + RTC_DCHECK_GE(num_channels, 1); + } + + AudioDecoderPcmA(const AudioDecoderPcmA&) = delete; + AudioDecoderPcmA& operator=(const AudioDecoderPcmA&) = delete; + + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + const size_t num_channels_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_G711_AUDIO_DECODER_PCM_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc b/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc new file mode 100644 index 0000000000..65e2da479d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" + +#include + +#include "modules/audio_coding/codecs/g711/g711_interface.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +bool AudioEncoderPcm::Config::IsOk() const { + return (frame_size_ms % 10 == 0) && (num_channels >= 1); +} + +AudioEncoderPcm::AudioEncoderPcm(const Config& config, int sample_rate_hz) + : sample_rate_hz_(sample_rate_hz), + num_channels_(config.num_channels), + payload_type_(config.payload_type), + num_10ms_frames_per_packet_( + static_cast(config.frame_size_ms / 10)), + full_frame_samples_(config.num_channels * config.frame_size_ms * + sample_rate_hz / 1000), + first_timestamp_in_buffer_(0) { + RTC_CHECK_GT(sample_rate_hz, 0) << "Sample rate must be larger than 0 Hz"; + RTC_CHECK_EQ(config.frame_size_ms % 10, 0) + << "Frame size must be an integer multiple of 10 ms."; + speech_buffer_.reserve(full_frame_samples_); +} + +AudioEncoderPcm::~AudioEncoderPcm() = default; + +int AudioEncoderPcm::SampleRateHz() const { + return sample_rate_hz_; +} + +size_t AudioEncoderPcm::NumChannels() const { + return num_channels_; +} + +size_t AudioEncoderPcm::Num10MsFramesInNextPacket() const { + return num_10ms_frames_per_packet_; +} + +size_t AudioEncoderPcm::Max10MsFramesInAPacket() const { + return num_10ms_frames_per_packet_; +} + +int AudioEncoderPcm::GetTargetBitrate() const { + return static_cast(8 * BytesPerSample() * SampleRateHz() * + NumChannels()); +} + +AudioEncoder::EncodedInfo AudioEncoderPcm::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + if (speech_buffer_.empty()) { + first_timestamp_in_buffer_ = rtp_timestamp; + } + speech_buffer_.insert(speech_buffer_.end(), audio.begin(), audio.end()); + if (speech_buffer_.size() < full_frame_samples_) { + return EncodedInfo(); + } + RTC_CHECK_EQ(speech_buffer_.size(), full_frame_samples_); + EncodedInfo info; + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.encoded_bytes = encoded->AppendData( + full_frame_samples_ * BytesPerSample(), + [&](rtc::ArrayView encoded) { + return EncodeCall(&speech_buffer_[0], full_frame_samples_, + encoded.data()); + }); + speech_buffer_.clear(); + info.encoder_type = GetCodecType(); + return info; +} + +void AudioEncoderPcm::Reset() { + speech_buffer_.clear(); +} + +absl::optional> +AudioEncoderPcm::GetFrameLengthRange() const { + return {{TimeDelta::Millis(num_10ms_frames_per_packet_ * 10), + TimeDelta::Millis(num_10ms_frames_per_packet_ * 10)}}; +} + +size_t AudioEncoderPcmA::EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) { + return WebRtcG711_EncodeA(audio, input_len, encoded); +} + +size_t AudioEncoderPcmA::BytesPerSample() const { + return 1; +} + +AudioEncoder::CodecType AudioEncoderPcmA::GetCodecType() const { + return AudioEncoder::CodecType::kPcmA; +} + +size_t AudioEncoderPcmU::EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) { + return WebRtcG711_EncodeU(audio, input_len, encoded); +} + +size_t AudioEncoderPcmU::BytesPerSample() const { + return 1; +} + +AudioEncoder::CodecType AudioEncoderPcmU::GetCodecType() const { + return AudioEncoder::CodecType::kPcmU; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.h b/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.h new file mode 100644 index 0000000000..d50be4b457 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.h @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G711_AUDIO_ENCODER_PCM_H_ +#define MODULES_AUDIO_CODING_CODECS_G711_AUDIO_ENCODER_PCM_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +class AudioEncoderPcm : public AudioEncoder { + public: + struct Config { + public: + bool IsOk() const; + + int frame_size_ms; + size_t num_channels; + int payload_type; + + protected: + explicit Config(int pt) + : frame_size_ms(20), num_channels(1), payload_type(pt) {} + }; + + ~AudioEncoderPcm() override; + + int SampleRateHz() const override; + size_t NumChannels() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + void Reset() override; + absl::optional> GetFrameLengthRange() + const override; + + protected: + AudioEncoderPcm(const Config& config, int sample_rate_hz); + + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + virtual size_t EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) = 0; + + virtual size_t BytesPerSample() const = 0; + + // Used to set EncodedInfoLeaf::encoder_type in + // AudioEncoderPcm::EncodeImpl + virtual AudioEncoder::CodecType GetCodecType() const = 0; + + private: + const int sample_rate_hz_; + const size_t num_channels_; + const int payload_type_; + const size_t num_10ms_frames_per_packet_; + const size_t full_frame_samples_; + std::vector speech_buffer_; + uint32_t first_timestamp_in_buffer_; +}; + +class AudioEncoderPcmA final : public AudioEncoderPcm { + public: + struct Config : public AudioEncoderPcm::Config { + Config() : AudioEncoderPcm::Config(8) {} + }; + + explicit AudioEncoderPcmA(const Config& config) + : AudioEncoderPcm(config, kSampleRateHz) {} + + AudioEncoderPcmA(const AudioEncoderPcmA&) = delete; + AudioEncoderPcmA& operator=(const AudioEncoderPcmA&) = delete; + + protected: + size_t EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) override; + + size_t BytesPerSample() const override; + + AudioEncoder::CodecType GetCodecType() const override; + + private: + static const int kSampleRateHz = 8000; +}; + +class AudioEncoderPcmU final : public AudioEncoderPcm { + public: + struct Config : public AudioEncoderPcm::Config { + Config() : AudioEncoderPcm::Config(0) {} + }; + + explicit AudioEncoderPcmU(const Config& config) + : AudioEncoderPcm(config, kSampleRateHz) {} + + AudioEncoderPcmU(const AudioEncoderPcmU&) = delete; + AudioEncoderPcmU& operator=(const AudioEncoderPcmU&) = delete; + + protected: + size_t EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) override; + + size_t BytesPerSample() const override; + + AudioEncoder::CodecType GetCodecType() const override; + + private: + static const int kSampleRateHz = 8000; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_G711_AUDIO_ENCODER_PCM_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.c b/third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.c new file mode 100644 index 0000000000..5fe1692ccb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/third_party/g711/g711.h" +#include "modules/audio_coding/codecs/g711/g711_interface.h" + +size_t WebRtcG711_EncodeA(const int16_t* speechIn, + size_t len, + uint8_t* encoded) { + size_t n; + for (n = 0; n < len; n++) + encoded[n] = linear_to_alaw(speechIn[n]); + return len; +} + +size_t WebRtcG711_EncodeU(const int16_t* speechIn, + size_t len, + uint8_t* encoded) { + size_t n; + for (n = 0; n < len; n++) + encoded[n] = linear_to_ulaw(speechIn[n]); + return len; +} + +size_t WebRtcG711_DecodeA(const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) { + size_t n; + for (n = 0; n < len; n++) + decoded[n] = alaw_to_linear(encoded[n]); + *speechType = 1; + return len; +} + +size_t WebRtcG711_DecodeU(const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) { + size_t n; + for (n = 0; n < len; n++) + decoded[n] = ulaw_to_linear(encoded[n]); + *speechType = 1; + return len; +} + +int16_t WebRtcG711_Version(char* version, int16_t lenBytes) { + strncpy(version, "2.0.0", lenBytes); + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.h b/third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.h new file mode 100644 index 0000000000..c92e6cc1c8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.h @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G711_G711_INTERFACE_H_ +#define MODULES_AUDIO_CODING_CODECS_G711_G711_INTERFACE_H_ + +#include +#include + +// Comfort noise constants +#define G711_WEBRTC_SPEECH 1 +#define G711_WEBRTC_CNG 2 + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * WebRtcG711_EncodeA(...) + * + * This function encodes a G711 A-law frame and inserts it into a packet. + * Input speech length has be of any length. + * + * Input: + * - speechIn : Input speech vector + * - len : Samples in speechIn + * + * Output: + * - encoded : The encoded data vector + * + * Return value : Length (in bytes) of coded data. + * Always equal to len input parameter. + */ + +size_t WebRtcG711_EncodeA(const int16_t* speechIn, + size_t len, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcG711_EncodeU(...) + * + * This function encodes a G711 U-law frame and inserts it into a packet. + * Input speech length has be of any length. + * + * Input: + * - speechIn : Input speech vector + * - len : Samples in speechIn + * + * Output: + * - encoded : The encoded data vector + * + * Return value : Length (in bytes) of coded data. + * Always equal to len input parameter. + */ + +size_t WebRtcG711_EncodeU(const int16_t* speechIn, + size_t len, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcG711_DecodeA(...) + * + * This function decodes a packet G711 A-law frame. + * + * Input: + * - encoded : Encoded data + * - len : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - speechType : 1 normal, 2 CNG (for G711 it should + * always return 1 since G711 does not have a + * built-in DTX/CNG scheme) + * + * Return value : >0 - Samples in decoded vector + * -1 - Error + */ + +size_t WebRtcG711_DecodeA(const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + +/**************************************************************************** + * WebRtcG711_DecodeU(...) + * + * This function decodes a packet G711 U-law frame. + * + * Input: + * - encoded : Encoded data + * - len : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - speechType : 1 normal, 2 CNG (for G711 it should + * always return 1 since G711 does not have a + * built-in DTX/CNG scheme) + * + * Return value : >0 - Samples in decoded vector + * -1 - Error + */ + +size_t WebRtcG711_DecodeU(const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + +/********************************************************************** + * WebRtcG711_Version(...) + * + * This function gives the version string of the G.711 codec. + * + * Input: + * - lenBytes: the size of Allocated space (in Bytes) where + * the version number is written to (in string format). + * + * Output: + * - version: Pointer to a buffer where the version number is + * written to. + * + */ + +int16_t WebRtcG711_Version(char* version, int16_t lenBytes); + +#ifdef __cplusplus +} +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_G711_G711_INTERFACE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g711/test/testG711.cc b/third_party/libwebrtc/modules/audio_coding/codecs/g711/test/testG711.cc new file mode 100644 index 0000000000..f3a42f5d79 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g711/test/testG711.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * testG711.cpp : Defines the entry point for the console application. + */ + +#include +#include +#include + +/* include API */ +#include "modules/audio_coding/codecs/g711/g711_interface.h" + +/* Runtime statistics */ +#include +#define CLOCKS_PER_SEC_G711 1000 + +/* function for reading audio data from PCM file */ +bool readframe(int16_t* data, FILE* inp, size_t length) { + size_t rlen = fread(data, sizeof(int16_t), length, inp); + if (rlen >= length) + return false; + memset(data + rlen, 0, (length - rlen) * sizeof(int16_t)); + return true; +} + +int main(int argc, char* argv[]) { + char inname[80], outname[40], bitname[40]; + FILE* inp; + FILE* outp; + FILE* bitp = NULL; + int framecnt; + bool endfile; + + size_t framelength = 80; + + /* Runtime statistics */ + double starttime; + double runtime; + double length_file; + + size_t stream_len = 0; + int16_t shortdata[480]; + int16_t decoded[480]; + uint8_t streamdata[1000]; + int16_t speechType[1]; + char law[2]; + char versionNumber[40]; + + /* handling wrong input arguments in the command line */ + if ((argc != 5) && (argc != 6)) { + printf("\n\nWrong number of arguments or flag values.\n\n"); + + printf("\n"); + printf("\nG.711 test application\n\n"); + printf("Usage:\n\n"); + printf("./testG711.exe framelength law infile outfile \n\n"); + printf("framelength: Framelength in samples.\n"); + printf("law : Coding law, A och u.\n"); + printf("infile : Normal speech input file\n"); + printf("outfile : Speech output file\n\n"); + printf("outbits : Output bitstream file [optional]\n\n"); + exit(0); + } + + /* Get version and print */ + WebRtcG711_Version(versionNumber, 40); + + printf("-----------------------------------\n"); + printf("G.711 version: %s\n\n", versionNumber); + /* Get frame length */ + int framelength_int = atoi(argv[1]); + if (framelength_int < 0) { + printf(" G.722: Invalid framelength %d.\n", framelength_int); + exit(1); + } + framelength = static_cast(framelength_int); + + /* Get compression law */ + strcpy(law, argv[2]); + + /* Get Input and Output files */ + sscanf(argv[3], "%s", inname); + sscanf(argv[4], "%s", outname); + if (argc == 6) { + sscanf(argv[5], "%s", bitname); + if ((bitp = fopen(bitname, "wb")) == NULL) { + printf(" G.711: Cannot read file %s.\n", bitname); + exit(1); + } + } + + if ((inp = fopen(inname, "rb")) == NULL) { + printf(" G.711: Cannot read file %s.\n", inname); + exit(1); + } + if ((outp = fopen(outname, "wb")) == NULL) { + printf(" G.711: Cannot write file %s.\n", outname); + exit(1); + } + printf("\nInput: %s\nOutput: %s\n", inname, outname); + if (argc == 6) { + printf("\nBitfile: %s\n", bitname); + } + + starttime = clock() / (double)CLOCKS_PER_SEC_G711; /* Runtime statistics */ + + /* Initialize encoder and decoder */ + framecnt = 0; + endfile = false; + while (!endfile) { + framecnt++; + /* Read speech block */ + endfile = readframe(shortdata, inp, framelength); + + /* G.711 encoding */ + if (!strcmp(law, "A")) { + /* A-law encoding */ + stream_len = WebRtcG711_EncodeA(shortdata, framelength, streamdata); + if (argc == 6) { + /* Write bits to file */ + if (fwrite(streamdata, sizeof(unsigned char), stream_len, bitp) != + stream_len) { + return -1; + } + } + WebRtcG711_DecodeA(streamdata, stream_len, decoded, speechType); + } else if (!strcmp(law, "u")) { + /* u-law encoding */ + stream_len = WebRtcG711_EncodeU(shortdata, framelength, streamdata); + if (argc == 6) { + /* Write bits to file */ + if (fwrite(streamdata, sizeof(unsigned char), stream_len, bitp) != + stream_len) { + return -1; + } + } + WebRtcG711_DecodeU(streamdata, stream_len, decoded, speechType); + } else { + printf("Wrong law mode\n"); + exit(1); + } + /* Write coded speech to file */ + if (fwrite(decoded, sizeof(short), framelength, outp) != framelength) { + return -1; + } + } + + runtime = (double)(clock() / (double)CLOCKS_PER_SEC_G711 - starttime); + length_file = ((double)framecnt * (double)framelength / 8000); + printf("\n\nLength of speech file: %.1f s\n", length_file); + printf("Time to run G.711: %.2f s (%.2f %% of realtime)\n\n", runtime, + (100 * runtime / length_file)); + printf("---------------------END----------------------\n"); + + fclose(inp); + fclose(outp); + + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc b/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc new file mode 100644 index 0000000000..1ecc9bc3d1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/g722/audio_decoder_g722.h" + +#include + +#include + +#include "modules/audio_coding/codecs/g722/g722_interface.h" +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +AudioDecoderG722Impl::AudioDecoderG722Impl() { + WebRtcG722_CreateDecoder(&dec_state_); + WebRtcG722_DecoderInit(dec_state_); +} + +AudioDecoderG722Impl::~AudioDecoderG722Impl() { + WebRtcG722_FreeDecoder(dec_state_); +} + +bool AudioDecoderG722Impl::HasDecodePlc() const { + return false; +} + +int AudioDecoderG722Impl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(SampleRateHz(), sample_rate_hz); + int16_t temp_type = 1; // Default is speech. + size_t ret = + WebRtcG722_Decode(dec_state_, encoded, encoded_len, decoded, &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return static_cast(ret); +} + +void AudioDecoderG722Impl::Reset() { + WebRtcG722_DecoderInit(dec_state_); +} + +std::vector AudioDecoderG722Impl::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), + timestamp, 8, 16); +} + +int AudioDecoderG722Impl::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // 1/2 encoded byte per sample per channel. + return static_cast(2 * encoded_len / Channels()); +} + +int AudioDecoderG722Impl::SampleRateHz() const { + return 16000; +} + +size_t AudioDecoderG722Impl::Channels() const { + return 1; +} + +AudioDecoderG722StereoImpl::AudioDecoderG722StereoImpl() { + WebRtcG722_CreateDecoder(&dec_state_left_); + WebRtcG722_CreateDecoder(&dec_state_right_); + WebRtcG722_DecoderInit(dec_state_left_); + WebRtcG722_DecoderInit(dec_state_right_); +} + +AudioDecoderG722StereoImpl::~AudioDecoderG722StereoImpl() { + WebRtcG722_FreeDecoder(dec_state_left_); + WebRtcG722_FreeDecoder(dec_state_right_); +} + +int AudioDecoderG722StereoImpl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(SampleRateHz(), sample_rate_hz); + // Adjust the encoded length down to ensure the same number of samples in each + // channel. + const size_t encoded_len_adjusted = PacketDuration(encoded, encoded_len) * + Channels() / + 2; // 1/2 byte per sample per channel + int16_t temp_type = 1; // Default is speech. + // De-interleave the bit-stream into two separate payloads. + uint8_t* encoded_deinterleaved = new uint8_t[encoded_len_adjusted]; + SplitStereoPacket(encoded, encoded_len_adjusted, encoded_deinterleaved); + // Decode left and right. + size_t decoded_len = + WebRtcG722_Decode(dec_state_left_, encoded_deinterleaved, + encoded_len_adjusted / 2, decoded, &temp_type); + size_t ret = WebRtcG722_Decode( + dec_state_right_, &encoded_deinterleaved[encoded_len_adjusted / 2], + encoded_len_adjusted / 2, &decoded[decoded_len], &temp_type); + if (ret == decoded_len) { + ret += decoded_len; // Return total number of samples. + // Interleave output. + for (size_t k = ret / 2; k < ret; k++) { + int16_t temp = decoded[k]; + memmove(&decoded[2 * k - ret + 2], &decoded[2 * k - ret + 1], + (ret - k - 1) * sizeof(int16_t)); + decoded[2 * k - ret + 1] = temp; + } + } + *speech_type = ConvertSpeechType(temp_type); + delete[] encoded_deinterleaved; + return static_cast(ret); +} + +int AudioDecoderG722StereoImpl::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // 1/2 encoded byte per sample per channel. Make sure the length represents + // an equal number of bytes per channel. Otherwise, we cannot de-interleave + // the encoded data later. + return static_cast(2 * (encoded_len / Channels())); +} + +int AudioDecoderG722StereoImpl::SampleRateHz() const { + return 16000; +} + +size_t AudioDecoderG722StereoImpl::Channels() const { + return 2; +} + +void AudioDecoderG722StereoImpl::Reset() { + WebRtcG722_DecoderInit(dec_state_left_); + WebRtcG722_DecoderInit(dec_state_right_); +} + +std::vector AudioDecoderG722StereoImpl::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + return LegacyEncodedAudioFrame::SplitBySamples(this, std::move(payload), + timestamp, 2 * 8, 16); +} + +// Split the stereo packet and place left and right channel after each other +// in the output array. +void AudioDecoderG722StereoImpl::SplitStereoPacket( + const uint8_t* encoded, + size_t encoded_len, + uint8_t* encoded_deinterleaved) { + // Regroup the 4 bits/sample so |l1 l2| |r1 r2| |l3 l4| |r3 r4| ..., + // where "lx" is 4 bits representing left sample number x, and "rx" right + // sample. Two samples fit in one byte, represented with |...|. + for (size_t i = 0; i + 1 < encoded_len; i += 2) { + uint8_t right_byte = ((encoded[i] & 0x0F) << 4) + (encoded[i + 1] & 0x0F); + encoded_deinterleaved[i] = (encoded[i] & 0xF0) + (encoded[i + 1] >> 4); + encoded_deinterleaved[i + 1] = right_byte; + } + + // Move one byte representing right channel each loop, and place it at the + // end of the bytestream vector. After looping the data is reordered to: + // |l1 l2| |l3 l4| ... |l(N-1) lN| |r1 r2| |r3 r4| ... |r(N-1) r(N)|, + // where N is the total number of samples. + for (size_t i = 0; i < encoded_len / 2; i++) { + uint8_t right_byte = encoded_deinterleaved[i + 1]; + memmove(&encoded_deinterleaved[i + 1], &encoded_deinterleaved[i + 2], + encoded_len - i - 2); + encoded_deinterleaved[encoded_len - 1] = right_byte; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h b/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h new file mode 100644 index 0000000000..5872fad5de --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G722_AUDIO_DECODER_G722_H_ +#define MODULES_AUDIO_CODING_CODECS_G722_AUDIO_DECODER_G722_H_ + +#include "api/audio_codecs/audio_decoder.h" + +typedef struct WebRtcG722DecInst G722DecInst; + +namespace webrtc { + +class AudioDecoderG722Impl final : public AudioDecoder { + public: + AudioDecoderG722Impl(); + ~AudioDecoderG722Impl() override; + + AudioDecoderG722Impl(const AudioDecoderG722Impl&) = delete; + AudioDecoderG722Impl& operator=(const AudioDecoderG722Impl&) = delete; + + bool HasDecodePlc() const override; + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + G722DecInst* dec_state_; +}; + +class AudioDecoderG722StereoImpl final : public AudioDecoder { + public: + AudioDecoderG722StereoImpl(); + ~AudioDecoderG722StereoImpl() override; + + AudioDecoderG722StereoImpl(const AudioDecoderG722StereoImpl&) = delete; + AudioDecoderG722StereoImpl& operator=(const AudioDecoderG722StereoImpl&) = + delete; + + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int SampleRateHz() const override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + // Splits the stereo-interleaved payload in `encoded` into separate payloads + // for left and right channels. The separated payloads are written to + // `encoded_deinterleaved`, which must hold at least `encoded_len` samples. + // The left channel starts at offset 0, while the right channel starts at + // offset encoded_len / 2 into `encoded_deinterleaved`. + void SplitStereoPacket(const uint8_t* encoded, + size_t encoded_len, + uint8_t* encoded_deinterleaved); + + G722DecInst* dec_state_left_; + G722DecInst* dec_state_right_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_G722_AUDIO_DECODER_G722_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc b/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc new file mode 100644 index 0000000000..b7d34ba581 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/g722/audio_encoder_g722.h" + +#include + +#include "modules/audio_coding/codecs/g722/g722_interface.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +namespace { + +const size_t kSampleRateHz = 16000; + +} // namespace + +AudioEncoderG722Impl::AudioEncoderG722Impl(const AudioEncoderG722Config& config, + int payload_type) + : num_channels_(config.num_channels), + payload_type_(payload_type), + num_10ms_frames_per_packet_( + static_cast(config.frame_size_ms / 10)), + num_10ms_frames_buffered_(0), + first_timestamp_in_buffer_(0), + encoders_(new EncoderState[num_channels_]), + interleave_buffer_(2 * num_channels_) { + RTC_CHECK(config.IsOk()); + const size_t samples_per_channel = + kSampleRateHz / 100 * num_10ms_frames_per_packet_; + for (size_t i = 0; i < num_channels_; ++i) { + encoders_[i].speech_buffer.reset(new int16_t[samples_per_channel]); + encoders_[i].encoded_buffer.SetSize(samples_per_channel / 2); + } + Reset(); +} + +AudioEncoderG722Impl::~AudioEncoderG722Impl() = default; + +int AudioEncoderG722Impl::SampleRateHz() const { + return kSampleRateHz; +} + +size_t AudioEncoderG722Impl::NumChannels() const { + return num_channels_; +} + +int AudioEncoderG722Impl::RtpTimestampRateHz() const { + // The RTP timestamp rate for G.722 is 8000 Hz, even though it is a 16 kHz + // codec. + return kSampleRateHz / 2; +} + +size_t AudioEncoderG722Impl::Num10MsFramesInNextPacket() const { + return num_10ms_frames_per_packet_; +} + +size_t AudioEncoderG722Impl::Max10MsFramesInAPacket() const { + return num_10ms_frames_per_packet_; +} + +int AudioEncoderG722Impl::GetTargetBitrate() const { + // 4 bits/sample, 16000 samples/s/channel. + return static_cast(64000 * NumChannels()); +} + +void AudioEncoderG722Impl::Reset() { + num_10ms_frames_buffered_ = 0; + for (size_t i = 0; i < num_channels_; ++i) + RTC_CHECK_EQ(0, WebRtcG722_EncoderInit(encoders_[i].encoder)); +} + +absl::optional> +AudioEncoderG722Impl::GetFrameLengthRange() const { + return {{TimeDelta::Millis(num_10ms_frames_per_packet_ * 10), + TimeDelta::Millis(num_10ms_frames_per_packet_ * 10)}}; +} + +AudioEncoder::EncodedInfo AudioEncoderG722Impl::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + if (num_10ms_frames_buffered_ == 0) + first_timestamp_in_buffer_ = rtp_timestamp; + + // Deinterleave samples and save them in each channel's buffer. + const size_t start = kSampleRateHz / 100 * num_10ms_frames_buffered_; + for (size_t i = 0; i < kSampleRateHz / 100; ++i) + for (size_t j = 0; j < num_channels_; ++j) + encoders_[j].speech_buffer[start + i] = audio[i * num_channels_ + j]; + + // If we don't yet have enough samples for a packet, we're done for now. + if (++num_10ms_frames_buffered_ < num_10ms_frames_per_packet_) { + return EncodedInfo(); + } + + // Encode each channel separately. + RTC_CHECK_EQ(num_10ms_frames_buffered_, num_10ms_frames_per_packet_); + num_10ms_frames_buffered_ = 0; + const size_t samples_per_channel = SamplesPerChannel(); + for (size_t i = 0; i < num_channels_; ++i) { + const size_t bytes_encoded = WebRtcG722_Encode( + encoders_[i].encoder, encoders_[i].speech_buffer.get(), + samples_per_channel, encoders_[i].encoded_buffer.data()); + RTC_CHECK_EQ(bytes_encoded, samples_per_channel / 2); + } + + const size_t bytes_to_encode = samples_per_channel / 2 * num_channels_; + EncodedInfo info; + info.encoded_bytes = encoded->AppendData( + bytes_to_encode, [&](rtc::ArrayView encoded) { + // Interleave the encoded bytes of the different channels. Each separate + // channel and the interleaved stream encodes two samples per byte, most + // significant half first. + for (size_t i = 0; i < samples_per_channel / 2; ++i) { + for (size_t j = 0; j < num_channels_; ++j) { + uint8_t two_samples = encoders_[j].encoded_buffer.data()[i]; + interleave_buffer_.data()[j] = two_samples >> 4; + interleave_buffer_.data()[num_channels_ + j] = two_samples & 0xf; + } + for (size_t j = 0; j < num_channels_; ++j) + encoded[i * num_channels_ + j] = + interleave_buffer_.data()[2 * j] << 4 | + interleave_buffer_.data()[2 * j + 1]; + } + + return bytes_to_encode; + }); + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.encoder_type = CodecType::kG722; + return info; +} + +AudioEncoderG722Impl::EncoderState::EncoderState() { + RTC_CHECK_EQ(0, WebRtcG722_CreateEncoder(&encoder)); +} + +AudioEncoderG722Impl::EncoderState::~EncoderState() { + RTC_CHECK_EQ(0, WebRtcG722_FreeEncoder(encoder)); +} + +size_t AudioEncoderG722Impl::SamplesPerChannel() const { + return kSampleRateHz / 100 * num_10ms_frames_per_packet_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.h b/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.h new file mode 100644 index 0000000000..a932aa8b7d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G722_AUDIO_ENCODER_G722_H_ +#define MODULES_AUDIO_CODING_CODECS_G722_AUDIO_ENCODER_G722_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/g722/audio_encoder_g722_config.h" +#include "api/units/time_delta.h" +#include "modules/audio_coding/codecs/g722/g722_interface.h" +#include "rtc_base/buffer.h" + +namespace webrtc { + +class AudioEncoderG722Impl final : public AudioEncoder { + public: + AudioEncoderG722Impl(const AudioEncoderG722Config& config, int payload_type); + ~AudioEncoderG722Impl() override; + + AudioEncoderG722Impl(const AudioEncoderG722Impl&) = delete; + AudioEncoderG722Impl& operator=(const AudioEncoderG722Impl&) = delete; + + int SampleRateHz() const override; + size_t NumChannels() const override; + int RtpTimestampRateHz() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + void Reset() override; + absl::optional> GetFrameLengthRange() + const override; + + protected: + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + private: + // The encoder state for one channel. + struct EncoderState { + G722EncInst* encoder; + std::unique_ptr speech_buffer; // Queued up for encoding. + rtc::Buffer encoded_buffer; // Already encoded. + EncoderState(); + ~EncoderState(); + }; + + size_t SamplesPerChannel() const; + + const size_t num_channels_; + const int payload_type_; + const size_t num_10ms_frames_per_packet_; + size_t num_10ms_frames_buffered_; + uint32_t first_timestamp_in_buffer_; + const std::unique_ptr encoders_; + rtc::Buffer interleave_buffer_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_G722_AUDIO_ENCODER_G722_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.c b/third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.c new file mode 100644 index 0000000000..36ee6d92be --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/audio_coding/codecs/g722/g722_interface.h" +#include "modules/third_party/g722/g722_enc_dec.h" + +int16_t WebRtcG722_CreateEncoder(G722EncInst **G722enc_inst) +{ + *G722enc_inst=(G722EncInst*)malloc(sizeof(G722EncoderState)); + if (*G722enc_inst!=NULL) { + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcG722_EncoderInit(G722EncInst *G722enc_inst) +{ + // Create and/or reset the G.722 encoder + // Bitrate 64 kbps and wideband mode (2) + G722enc_inst = (G722EncInst *) WebRtc_g722_encode_init( + (G722EncoderState*) G722enc_inst, 64000, 2); + if (G722enc_inst == NULL) { + return -1; + } else { + return 0; + } +} + +int WebRtcG722_FreeEncoder(G722EncInst *G722enc_inst) +{ + // Free encoder memory + return WebRtc_g722_encode_release((G722EncoderState*) G722enc_inst); +} + +size_t WebRtcG722_Encode(G722EncInst *G722enc_inst, + const int16_t* speechIn, + size_t len, + uint8_t* encoded) +{ + unsigned char *codechar = (unsigned char*) encoded; + // Encode the input speech vector + return WebRtc_g722_encode((G722EncoderState*) G722enc_inst, codechar, + speechIn, len); +} + +int16_t WebRtcG722_CreateDecoder(G722DecInst **G722dec_inst) +{ + *G722dec_inst=(G722DecInst*)malloc(sizeof(G722DecoderState)); + if (*G722dec_inst!=NULL) { + return(0); + } else { + return(-1); + } +} + +void WebRtcG722_DecoderInit(G722DecInst* inst) { + // Create and/or reset the G.722 decoder + // Bitrate 64 kbps and wideband mode (2) + WebRtc_g722_decode_init((G722DecoderState*)inst, 64000, 2); +} + +int WebRtcG722_FreeDecoder(G722DecInst *G722dec_inst) +{ + // Free encoder memory + return WebRtc_g722_decode_release((G722DecoderState*) G722dec_inst); +} + +size_t WebRtcG722_Decode(G722DecInst *G722dec_inst, + const uint8_t *encoded, + size_t len, + int16_t *decoded, + int16_t *speechType) +{ + // Decode the G.722 encoder stream + *speechType=G722_WEBRTC_SPEECH; + return WebRtc_g722_decode((G722DecoderState*) G722dec_inst, decoded, + encoded, len); +} + +int16_t WebRtcG722_Version(char *versionStr, short len) +{ + // Get version string + char version[30] = "2.0.0\n"; + if (strlen(version) < (unsigned int)len) + { + strcpy(versionStr, version); + return 0; + } + else + { + return -1; + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.h b/third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.h new file mode 100644 index 0000000000..353de4504f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_G722_G722_INTERFACE_H_ +#define MODULES_AUDIO_CODING_CODECS_G722_G722_INTERFACE_H_ + +#include +#include + +/* + * Solution to support multiple instances + */ + +typedef struct WebRtcG722EncInst G722EncInst; +typedef struct WebRtcG722DecInst G722DecInst; + +/* + * Comfort noise constants + */ + +#define G722_WEBRTC_SPEECH 1 +#define G722_WEBRTC_CNG 2 + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * WebRtcG722_CreateEncoder(...) + * + * Create memory used for G722 encoder + * + * Input: + * - G722enc_inst : G722 instance for encoder + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcG722_CreateEncoder(G722EncInst** G722enc_inst); + +/**************************************************************************** + * WebRtcG722_EncoderInit(...) + * + * This function initializes a G722 instance + * + * Input: + * - G722enc_inst : G722 instance, i.e. the user that should receive + * be initialized + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcG722_EncoderInit(G722EncInst* G722enc_inst); + +/**************************************************************************** + * WebRtcG722_FreeEncoder(...) + * + * Free the memory used for G722 encoder + * + * Input: + * - G722enc_inst : G722 instance for encoder + * + * Return value : 0 - Ok + * -1 - Error + */ +int WebRtcG722_FreeEncoder(G722EncInst* G722enc_inst); + +/**************************************************************************** + * WebRtcG722_Encode(...) + * + * This function encodes G722 encoded data. + * + * Input: + * - G722enc_inst : G722 instance, i.e. the user that should encode + * a packet + * - speechIn : Input speech vector + * - len : Samples in speechIn + * + * Output: + * - encoded : The encoded data vector + * + * Return value : Length (in bytes) of coded data + */ + +size_t WebRtcG722_Encode(G722EncInst* G722enc_inst, + const int16_t* speechIn, + size_t len, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcG722_CreateDecoder(...) + * + * Create memory used for G722 encoder + * + * Input: + * - G722dec_inst : G722 instance for decoder + * + * Return value : 0 - Ok + * -1 - Error + */ +int16_t WebRtcG722_CreateDecoder(G722DecInst** G722dec_inst); + +/**************************************************************************** + * WebRtcG722_DecoderInit(...) + * + * This function initializes a G722 instance + * + * Input: + * - inst : G722 instance + */ + +void WebRtcG722_DecoderInit(G722DecInst* inst); + +/**************************************************************************** + * WebRtcG722_FreeDecoder(...) + * + * Free the memory used for G722 decoder + * + * Input: + * - G722dec_inst : G722 instance for decoder + * + * Return value : 0 - Ok + * -1 - Error + */ + +int WebRtcG722_FreeDecoder(G722DecInst* G722dec_inst); + +/**************************************************************************** + * WebRtcG722_Decode(...) + * + * This function decodes a packet with G729 frame(s). Output speech length + * will be a multiple of 80 samples (80*frames/packet). + * + * Input: + * - G722dec_inst : G722 instance, i.e. the user that should decode + * a packet + * - encoded : Encoded G722 frame(s) + * - len : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - speechType : 1 normal, 2 CNG (Since G722 does not have its own + * DTX/CNG scheme it should always return 1) + * + * Return value : Samples in decoded vector + */ + +size_t WebRtcG722_Decode(G722DecInst* G722dec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + +/**************************************************************************** + * WebRtcG722_Version(...) + * + * Get a string with the current version of the codec + */ + +int16_t WebRtcG722_Version(char* versionStr, short len); + +#ifdef __cplusplus +} +#endif + +#endif /* MODULES_AUDIO_CODING_CODECS_G722_G722_INTERFACE_H_ */ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/g722/test/testG722.cc b/third_party/libwebrtc/modules/audio_coding/codecs/g722/test/testG722.cc new file mode 100644 index 0000000000..9f2155d0f7 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/g722/test/testG722.cc @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * testG722.cpp : Defines the entry point for the console application. + */ + +#include +#include +#include + +/* include API */ +#include "modules/audio_coding/codecs/g722/g722_interface.h" + +/* Runtime statistics */ +#include +#define CLOCKS_PER_SEC_G722 100000 + +// Forward declaration +typedef struct WebRtcG722EncInst G722EncInst; +typedef struct WebRtcG722DecInst G722DecInst; + +/* function for reading audio data from PCM file */ +bool readframe(int16_t* data, FILE* inp, size_t length) { + size_t rlen = fread(data, sizeof(int16_t), length, inp); + if (rlen >= length) + return false; + memset(data + rlen, 0, (length - rlen) * sizeof(int16_t)); + return true; +} + +int main(int argc, char* argv[]) { + char inname[60], outbit[40], outname[40]; + FILE *inp, *outbitp, *outp; + + int framecnt; + bool endfile; + size_t framelength = 160; + G722EncInst* G722enc_inst; + G722DecInst* G722dec_inst; + + /* Runtime statistics */ + double starttime; + double runtime = 0; + double length_file; + + size_t stream_len = 0; + int16_t shortdata[960]; + int16_t decoded[960]; + uint8_t streamdata[80 * 6]; + int16_t speechType[1]; + + /* handling wrong input arguments in the command line */ + if (argc != 5) { + printf("\n\nWrong number of arguments or flag values.\n\n"); + + printf("\n"); + printf("Usage:\n\n"); + printf("./testG722.exe framelength infile outbitfile outspeechfile \n\n"); + printf("with:\n"); + printf("framelength : Framelength in samples.\n\n"); + printf("infile : Normal speech input file\n\n"); + printf("outbitfile : Bitstream output file\n\n"); + printf("outspeechfile: Speech output file\n\n"); + exit(0); + } + + /* Get frame length */ + int framelength_int = atoi(argv[1]); + if (framelength_int < 0) { + printf(" G.722: Invalid framelength %d.\n", framelength_int); + exit(1); + } + framelength = static_cast(framelength_int); + + /* Get Input and Output files */ + sscanf(argv[2], "%s", inname); + sscanf(argv[3], "%s", outbit); + sscanf(argv[4], "%s", outname); + + if ((inp = fopen(inname, "rb")) == NULL) { + printf(" G.722: Cannot read file %s.\n", inname); + exit(1); + } + if ((outbitp = fopen(outbit, "wb")) == NULL) { + printf(" G.722: Cannot write file %s.\n", outbit); + exit(1); + } + if ((outp = fopen(outname, "wb")) == NULL) { + printf(" G.722: Cannot write file %s.\n", outname); + exit(1); + } + printf("\nInput:%s\nOutput bitstream:%s\nOutput:%s\n", inname, outbit, + outname); + + /* Create and init */ + WebRtcG722_CreateEncoder((G722EncInst**)&G722enc_inst); + WebRtcG722_CreateDecoder((G722DecInst**)&G722dec_inst); + WebRtcG722_EncoderInit((G722EncInst*)G722enc_inst); + WebRtcG722_DecoderInit((G722DecInst*)G722dec_inst); + + /* Initialize encoder and decoder */ + framecnt = 0; + endfile = false; + while (!endfile) { + framecnt++; + + /* Read speech block */ + endfile = readframe(shortdata, inp, framelength); + + /* Start clock before call to encoder and decoder */ + starttime = clock() / (double)CLOCKS_PER_SEC_G722; + + /* G.722 encoding + decoding */ + stream_len = WebRtcG722_Encode((G722EncInst*)G722enc_inst, shortdata, + framelength, streamdata); + WebRtcG722_Decode(G722dec_inst, streamdata, stream_len, decoded, + speechType); + + /* Stop clock after call to encoder and decoder */ + runtime += (double)((clock() / (double)CLOCKS_PER_SEC_G722) - starttime); + + /* Write coded bits to file */ + if (fwrite(streamdata, sizeof(short), stream_len / 2, outbitp) != + stream_len / 2) { + return -1; + } + /* Write coded speech to file */ + if (fwrite(decoded, sizeof(short), framelength, outp) != framelength) { + return -1; + } + } + + WebRtcG722_FreeEncoder((G722EncInst*)G722enc_inst); + WebRtcG722_FreeDecoder((G722DecInst*)G722dec_inst); + + length_file = ((double)framecnt * (double)framelength / 16000); + printf("\n\nLength of speech file: %.1f s\n", length_file); + printf("Time to run G.722: %.2f s (%.2f %% of realtime)\n\n", runtime, + (100 * runtime / length_file)); + printf("---------------------END----------------------\n"); + + fclose(inp); + fclose(outbitp); + fclose(outp); + + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.c new file mode 100644 index 0000000000..77da78ba7f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.c @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AbsQuant.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/abs_quant.h" + +#include "modules/audio_coding/codecs/ilbc/abs_quant_loop.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + + +/*----------------------------------------------------------------* + * predictive noise shaping encoding of scaled start state + * (subrutine for WebRtcIlbcfix_StateSearch) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_AbsQuant( + IlbcEncoder *iLBCenc_inst, + /* (i) Encoder instance */ + iLBC_bits *iLBC_encbits, /* (i/o) Encoded bits (outputs idxForMax + and idxVec, uses state_first as + input) */ + int16_t *in, /* (i) vector to encode */ + int16_t *weightDenum /* (i) denominator of synthesis filter */ + ) { + int16_t *syntOut; + size_t quantLen[2]; + + /* Stack based */ + int16_t syntOutBuf[LPC_FILTERORDER+STATE_SHORT_LEN_30MS]; + int16_t in_weightedVec[STATE_SHORT_LEN_30MS+LPC_FILTERORDER]; + int16_t *in_weighted = &in_weightedVec[LPC_FILTERORDER]; + + /* Initialize the buffers */ + WebRtcSpl_MemSetW16(syntOutBuf, 0, LPC_FILTERORDER+STATE_SHORT_LEN_30MS); + syntOut = &syntOutBuf[LPC_FILTERORDER]; + /* Start with zero state */ + WebRtcSpl_MemSetW16(in_weightedVec, 0, LPC_FILTERORDER); + + /* Perform the quantization loop in two sections of length quantLen[i], + where the perceptual weighting filter is updated at the subframe + border */ + + if (iLBC_encbits->state_first) { + quantLen[0]=SUBL; + quantLen[1]=iLBCenc_inst->state_short_len-SUBL; + } else { + quantLen[0]=iLBCenc_inst->state_short_len-SUBL; + quantLen[1]=SUBL; + } + + /* Calculate the weighted residual, switch perceptual weighting + filter at the subframe border */ + WebRtcSpl_FilterARFastQ12( + in, in_weighted, + weightDenum, LPC_FILTERORDER+1, quantLen[0]); + WebRtcSpl_FilterARFastQ12( + &in[quantLen[0]], &in_weighted[quantLen[0]], + &weightDenum[LPC_FILTERORDER+1], LPC_FILTERORDER+1, quantLen[1]); + + WebRtcIlbcfix_AbsQuantLoop( + syntOut, + in_weighted, + weightDenum, + quantLen, + iLBC_encbits->idxVec); + +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.h new file mode 100644 index 0000000000..c72e29cf29 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AbsQuant.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ABS_QUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ABS_QUANT_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * predictive noise shaping encoding of scaled start state + * (subrutine for WebRtcIlbcfix_StateSearch) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_AbsQuant( + IlbcEncoder* iLBCenc_inst, + /* (i) Encoder instance */ + iLBC_bits* iLBC_encbits, /* (i/o) Encoded bits (outputs idxForMax + and idxVec, uses state_first as + input) */ + int16_t* in, /* (i) vector to encode */ + int16_t* weightDenum /* (i) denominator of synthesis filter */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c new file mode 100644 index 0000000000..cf9266299d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AbsQuantLoop.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/abs_quant_loop.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/sort_sq.h" + +void WebRtcIlbcfix_AbsQuantLoop(int16_t *syntOutIN, int16_t *in_weightedIN, + int16_t *weightDenumIN, size_t *quantLenIN, + int16_t *idxVecIN ) { + size_t k1, k2; + int16_t index; + int32_t toQW32; + int32_t toQ32; + int16_t tmp16a; + int16_t xq; + + int16_t *syntOut = syntOutIN; + int16_t *in_weighted = in_weightedIN; + int16_t *weightDenum = weightDenumIN; + size_t *quantLen = quantLenIN; + int16_t *idxVec = idxVecIN; + + for(k1=0;k1<2;k1++) { + for(k2=0;k2 32767) { + toQ32 = (int32_t) 32767; + } else if (toQ32 < -32768) { + toQ32 = (int32_t) -32768; + } + + /* Quantize the state */ + if (toQW32<(-7577)) { + /* To prevent negative overflow */ + index=0; + } else if (toQW32>8151) { + /* To prevent positive overflow */ + index=7; + } else { + /* Find the best quantization index + (state_sq3Tbl is in Q13 and toQ is in Q11) + */ + WebRtcIlbcfix_SortSq(&xq, &index, + (int16_t)toQ32, + WebRtcIlbcfix_kStateSq3, 8); + } + + /* Store selected index */ + (*idxVec++) = index; + + /* Compute decoded sample and update of the prediction filter */ + tmp16a = ((WebRtcIlbcfix_kStateSq3[index] + 2 ) >> 2); + + *syntOut = (int16_t) (tmp16a + (int32_t)(*in_weighted) - toQW32); + + syntOut++; in_weighted++; + } + /* Update perceptual weighting filter at subframe border */ + weightDenum += 11; + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.h new file mode 100644 index 0000000000..841d73b9fb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AbsQuantLoop.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ABS_QUANT_LOOP_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ABS_QUANT_LOOP_H_ + +#include +#include + +/*----------------------------------------------------------------* + * predictive noise shaping encoding of scaled start state + * (subrutine for WebRtcIlbcfix_StateSearch) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_AbsQuantLoop(int16_t* syntOutIN, + int16_t* in_weightedIN, + int16_t* weightDenumIN, + size_t* quantLenIN, + int16_t* idxVecIN); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc new file mode 100644 index 0000000000..57b5abbe23 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/ilbc.h" +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +AudioDecoderIlbcImpl::AudioDecoderIlbcImpl() { + WebRtcIlbcfix_DecoderCreate(&dec_state_); + WebRtcIlbcfix_Decoderinit30Ms(dec_state_); +} + +AudioDecoderIlbcImpl::~AudioDecoderIlbcImpl() { + WebRtcIlbcfix_DecoderFree(dec_state_); +} + +bool AudioDecoderIlbcImpl::HasDecodePlc() const { + return true; +} + +int AudioDecoderIlbcImpl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(sample_rate_hz, 8000); + int16_t temp_type = 1; // Default is speech. + int ret = WebRtcIlbcfix_Decode(dec_state_, encoded, encoded_len, decoded, + &temp_type); + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +size_t AudioDecoderIlbcImpl::DecodePlc(size_t num_frames, int16_t* decoded) { + return WebRtcIlbcfix_NetEqPlc(dec_state_, decoded, num_frames); +} + +void AudioDecoderIlbcImpl::Reset() { + WebRtcIlbcfix_Decoderinit30Ms(dec_state_); +} + +std::vector AudioDecoderIlbcImpl::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + std::vector results; + size_t bytes_per_frame; + int timestamps_per_frame; + if (payload.size() >= 950) { + RTC_LOG(LS_WARNING) + << "AudioDecoderIlbcImpl::ParsePayload: Payload too large"; + return results; + } + if (payload.size() % 38 == 0) { + // 20 ms frames. + bytes_per_frame = 38; + timestamps_per_frame = 160; + } else if (payload.size() % 50 == 0) { + // 30 ms frames. + bytes_per_frame = 50; + timestamps_per_frame = 240; + } else { + RTC_LOG(LS_WARNING) + << "AudioDecoderIlbcImpl::ParsePayload: Invalid payload"; + return results; + } + + RTC_DCHECK_EQ(0, payload.size() % bytes_per_frame); + if (payload.size() == bytes_per_frame) { + std::unique_ptr frame( + new LegacyEncodedAudioFrame(this, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); + } else { + size_t byte_offset; + uint32_t timestamp_offset; + for (byte_offset = 0, timestamp_offset = 0; byte_offset < payload.size(); + byte_offset += bytes_per_frame, + timestamp_offset += timestamps_per_frame) { + std::unique_ptr frame(new LegacyEncodedAudioFrame( + this, rtc::Buffer(payload.data() + byte_offset, bytes_per_frame))); + results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame)); + } + } + + return results; +} + +int AudioDecoderIlbcImpl::SampleRateHz() const { + return 8000; +} + +size_t AudioDecoderIlbcImpl::Channels() const { + return 1; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h new file mode 100644 index 0000000000..46ba755148 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_DECODER_ILBC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_DECODER_ILBC_H_ + +#include +#include + +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/buffer.h" + +typedef struct iLBC_decinst_t_ IlbcDecoderInstance; + +namespace webrtc { + +class AudioDecoderIlbcImpl final : public AudioDecoder { + public: + AudioDecoderIlbcImpl(); + ~AudioDecoderIlbcImpl() override; + + AudioDecoderIlbcImpl(const AudioDecoderIlbcImpl&) = delete; + AudioDecoderIlbcImpl& operator=(const AudioDecoderIlbcImpl&) = delete; + + bool HasDecodePlc() const override; + size_t DecodePlc(size_t num_frames, int16_t* decoded) override; + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + IlbcDecoderInstance* dec_state_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_DECODER_ILBC_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc new file mode 100644 index 0000000000..9fbf42ceeb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/ilbc.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +namespace { + +const int kSampleRateHz = 8000; + +int GetIlbcBitrate(int ptime) { + switch (ptime) { + case 20: + case 40: + // 38 bytes per frame of 20 ms => 15200 bits/s. + return 15200; + case 30: + case 60: + // 50 bytes per frame of 30 ms => (approx) 13333 bits/s. + return 13333; + default: + RTC_CHECK_NOTREACHED(); + } +} + +} // namespace + +AudioEncoderIlbcImpl::AudioEncoderIlbcImpl(const AudioEncoderIlbcConfig& config, + int payload_type) + : frame_size_ms_(config.frame_size_ms), + payload_type_(payload_type), + num_10ms_frames_per_packet_( + static_cast(config.frame_size_ms / 10)), + encoder_(nullptr) { + RTC_CHECK(config.IsOk()); + Reset(); +} + +AudioEncoderIlbcImpl::~AudioEncoderIlbcImpl() { + RTC_CHECK_EQ(0, WebRtcIlbcfix_EncoderFree(encoder_)); +} + +int AudioEncoderIlbcImpl::SampleRateHz() const { + return kSampleRateHz; +} + +size_t AudioEncoderIlbcImpl::NumChannels() const { + return 1; +} + +size_t AudioEncoderIlbcImpl::Num10MsFramesInNextPacket() const { + return num_10ms_frames_per_packet_; +} + +size_t AudioEncoderIlbcImpl::Max10MsFramesInAPacket() const { + return num_10ms_frames_per_packet_; +} + +int AudioEncoderIlbcImpl::GetTargetBitrate() const { + return GetIlbcBitrate(rtc::dchecked_cast(num_10ms_frames_per_packet_) * + 10); +} + +AudioEncoder::EncodedInfo AudioEncoderIlbcImpl::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + // Save timestamp if starting a new packet. + if (num_10ms_frames_buffered_ == 0) + first_timestamp_in_buffer_ = rtp_timestamp; + + // Buffer input. + std::copy(audio.cbegin(), audio.cend(), + input_buffer_ + kSampleRateHz / 100 * num_10ms_frames_buffered_); + + // If we don't yet have enough buffered input for a whole packet, we're done + // for now. + if (++num_10ms_frames_buffered_ < num_10ms_frames_per_packet_) { + return EncodedInfo(); + } + + // Encode buffered input. + RTC_DCHECK_EQ(num_10ms_frames_buffered_, num_10ms_frames_per_packet_); + num_10ms_frames_buffered_ = 0; + size_t encoded_bytes = encoded->AppendData( + RequiredOutputSizeBytes(), [&](rtc::ArrayView encoded) { + const int r = WebRtcIlbcfix_Encode( + encoder_, input_buffer_, + kSampleRateHz / 100 * num_10ms_frames_per_packet_, encoded.data()); + RTC_CHECK_GE(r, 0); + + return static_cast(r); + }); + + RTC_DCHECK_EQ(encoded_bytes, RequiredOutputSizeBytes()); + + EncodedInfo info; + info.encoded_bytes = encoded_bytes; + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.encoder_type = CodecType::kIlbc; + return info; +} + +void AudioEncoderIlbcImpl::Reset() { + if (encoder_) + RTC_CHECK_EQ(0, WebRtcIlbcfix_EncoderFree(encoder_)); + RTC_CHECK_EQ(0, WebRtcIlbcfix_EncoderCreate(&encoder_)); + const int encoder_frame_size_ms = + frame_size_ms_ > 30 ? frame_size_ms_ / 2 : frame_size_ms_; + RTC_CHECK_EQ(0, WebRtcIlbcfix_EncoderInit(encoder_, encoder_frame_size_ms)); + num_10ms_frames_buffered_ = 0; +} + +absl::optional> +AudioEncoderIlbcImpl::GetFrameLengthRange() const { + return {{TimeDelta::Millis(num_10ms_frames_per_packet_ * 10), + TimeDelta::Millis(num_10ms_frames_per_packet_ * 10)}}; +} + +size_t AudioEncoderIlbcImpl::RequiredOutputSizeBytes() const { + switch (num_10ms_frames_per_packet_) { + case 2: + return 38; + case 3: + return 50; + case 4: + return 2 * 38; + case 6: + return 2 * 50; + default: + RTC_CHECK_NOTREACHED(); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h new file mode 100644 index 0000000000..c8dfa2ca6d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_ENCODER_ILBC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_ENCODER_ILBC_H_ + +#include +#include + +#include + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/ilbc/audio_encoder_ilbc_config.h" +#include "api/units/time_delta.h" +#include "modules/audio_coding/codecs/ilbc/ilbc.h" + +namespace webrtc { + +class AudioEncoderIlbcImpl final : public AudioEncoder { + public: + AudioEncoderIlbcImpl(const AudioEncoderIlbcConfig& config, int payload_type); + ~AudioEncoderIlbcImpl() override; + + AudioEncoderIlbcImpl(const AudioEncoderIlbcImpl&) = delete; + AudioEncoderIlbcImpl& operator=(const AudioEncoderIlbcImpl&) = delete; + + int SampleRateHz() const override; + size_t NumChannels() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + void Reset() override; + absl::optional> GetFrameLengthRange() + const override; + + private: + size_t RequiredOutputSizeBytes() const; + + static constexpr size_t kMaxSamplesPerPacket = 480; + const int frame_size_ms_; + const int payload_type_; + const size_t num_10ms_frames_per_packet_; + size_t num_10ms_frames_buffered_; + uint32_t first_timestamp_in_buffer_; + int16_t input_buffer_[kMaxSamplesPerPacket]; + IlbcEncoderInstance* encoder_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_CODECS_ILBC_AUDIO_ENCODER_ILBC_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c new file mode 100644 index 0000000000..c915a2f9f0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AugmentedCbCorr.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/augmented_cb_corr.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +void WebRtcIlbcfix_AugmentedCbCorr( + int16_t *target, /* (i) Target vector */ + int16_t *buffer, /* (i) Memory buffer */ + int16_t *interpSamples, /* (i) buffer with + interpolated samples */ + int32_t *crossDot, /* (o) The cross correlation between + the target and the Augmented + vector */ + size_t low, /* (i) Lag to start from (typically + 20) */ + size_t high, /* (i) Lag to end at (typically 39) */ + int scale) /* (i) Scale factor to use for + the crossDot */ +{ + size_t lagcount; + size_t ilow; + int16_t *targetPtr; + int32_t *crossDotPtr; + int16_t *iSPtr=interpSamples; + + /* Calculate the correlation between the target and the + interpolated codebook. The correlation is calculated in + 3 sections with the interpolated part in the middle */ + crossDotPtr=crossDot; + for (lagcount=low; lagcount<=high; lagcount++) { + + ilow = lagcount - 4; + + /* Compute dot product for the first (lagcount-4) samples */ + (*crossDotPtr) = WebRtcSpl_DotProductWithScale(target, buffer-lagcount, ilow, scale); + + /* Compute dot product on the interpolated samples */ + (*crossDotPtr) += WebRtcSpl_DotProductWithScale(target+ilow, iSPtr, 4, scale); + targetPtr = target + lagcount; + iSPtr += lagcount-ilow; + + /* Compute dot product for the remaining samples */ + (*crossDotPtr) += WebRtcSpl_DotProductWithScale(targetPtr, buffer-lagcount, SUBL-lagcount, scale); + crossDotPtr++; + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.h new file mode 100644 index 0000000000..2e9612e51a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_AugmentedCbCorr.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_AUGMENTED_CB_CORR_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_AUGMENTED_CB_CORR_H_ + +#include +#include + +/*----------------------------------------------------------------* + * Calculate correlation between target and Augmented codebooks + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_AugmentedCbCorr( + int16_t* target, /* (i) Target vector */ + int16_t* buffer, /* (i) Memory buffer */ + int16_t* interpSamples, /* (i) buffer with + interpolated samples */ + int32_t* crossDot, /* (o) The cross correlation between + the target and the Augmented + vector */ + size_t low, /* (i) Lag to start from (typically + 20) */ + size_t high, /* (i) Lag to end at (typically 39 */ + int scale); /* (i) Scale factor to use for the crossDot */ + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.c new file mode 100644 index 0000000000..1a9b882adf --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_BwExpand.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/bw_expand.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * lpc bandwidth expansion + *---------------------------------------------------------------*/ + +/* The output is in the same domain as the input */ +void WebRtcIlbcfix_BwExpand( + int16_t *out, /* (o) the bandwidth expanded lpc coefficients */ + int16_t *in, /* (i) the lpc coefficients before bandwidth + expansion */ + int16_t *coef, /* (i) the bandwidth expansion factor Q15 */ + int16_t length /* (i) the length of lpc coefficient vectors */ + ) { + int i; + + out[0] = in[0]; + for (i = 1; i < length; i++) { + /* out[i] = coef[i] * in[i] with rounding. + in[] and out[] are in Q12 and coef[] is in Q15 + */ + out[i] = (int16_t)((coef[i] * in[i] + 16384) >> 15); + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.h new file mode 100644 index 0000000000..ff9b0b302e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_BwExpand.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_BW_EXPAND_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_BW_EXPAND_H_ + +#include +#include + +/*----------------------------------------------------------------* + * lpc bandwidth expansion + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_BwExpand( + int16_t* out, /* (o) the bandwidth expanded lpc coefficients */ + int16_t* in, /* (i) the lpc coefficients before bandwidth + expansion */ + int16_t* coef, /* (i) the bandwidth expansion factor Q15 */ + int16_t length /* (i) the length of lpc coefficient vectors */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.c new file mode 100644 index 0000000000..1e9a7040c7 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbConstruct.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/cb_construct.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/gain_dequant.h" +#include "modules/audio_coding/codecs/ilbc/get_cd_vec.h" +#include "rtc_base/sanitizer.h" + +// An arithmetic operation that is allowed to overflow. (It's still undefined +// behavior, so not a good idea; this just makes UBSan ignore the violation, so +// that our old code can continue to do what it's always been doing.) +static inline int32_t RTC_NO_SANITIZE("signed-integer-overflow") + OverflowingAddS32S32ToS32(int32_t a, int32_t b) { + return a + b; +} + +/*----------------------------------------------------------------* + * Construct decoded vector from codebook and gains. + *---------------------------------------------------------------*/ + +bool WebRtcIlbcfix_CbConstruct( + int16_t* decvector, /* (o) Decoded vector */ + const int16_t* index, /* (i) Codebook indices */ + const int16_t* gain_index, /* (i) Gain quantization indices */ + int16_t* mem, /* (i) Buffer for codevector construction */ + size_t lMem, /* (i) Length of buffer */ + size_t veclen) { /* (i) Length of vector */ + size_t j; + int16_t gain[CB_NSTAGES]; + /* Stack based */ + int16_t cbvec0[SUBL]; + int16_t cbvec1[SUBL]; + int16_t cbvec2[SUBL]; + int32_t a32; + int16_t *gainPtr; + + /* gain de-quantization */ + + gain[0] = WebRtcIlbcfix_GainDequant(gain_index[0], 16384, 0); + gain[1] = WebRtcIlbcfix_GainDequant(gain_index[1], gain[0], 1); + gain[2] = WebRtcIlbcfix_GainDequant(gain_index[2], gain[1], 2); + + /* codebook vector construction and construction of total vector */ + + /* Stack based */ + if (!WebRtcIlbcfix_GetCbVec(cbvec0, mem, (size_t)index[0], lMem, veclen)) + return false; // Failure. + if (!WebRtcIlbcfix_GetCbVec(cbvec1, mem, (size_t)index[1], lMem, veclen)) + return false; // Failure. + if (!WebRtcIlbcfix_GetCbVec(cbvec2, mem, (size_t)index[2], lMem, veclen)) + return false; // Failure. + + gainPtr = &gain[0]; + for (j=0;j> 14); + } + + return true; // Success. +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.h new file mode 100644 index 0000000000..8f7c663164 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbConstruct.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_CONSTRUCT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_CONSTRUCT_H_ + +#include +#include +#include + +#include "absl/base/attributes.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Construct decoded vector from codebook and gains. + *---------------------------------------------------------------*/ + +// Returns true on success, false on failure. +ABSL_MUST_USE_RESULT +bool WebRtcIlbcfix_CbConstruct( + int16_t* decvector, /* (o) Decoded vector */ + const int16_t* index, /* (i) Codebook indices */ + const int16_t* gain_index, /* (i) Gain quantization indices */ + int16_t* mem, /* (i) Buffer for codevector construction */ + size_t lMem, /* (i) Length of buffer */ + size_t veclen /* (i) Length of vector */ +); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c new file mode 100644 index 0000000000..21e4197607 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergy.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/cb_mem_energy.h" + +#include "modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Function WebRtcIlbcfix_CbMemEnergy computes the energy of all + * the vectors in the codebook memory that will be used in the + * following search for the best match. + *----------------------------------------------------------------*/ + +void WebRtcIlbcfix_CbMemEnergy( + size_t range, + int16_t *CB, /* (i) The CB memory (1:st section) */ + int16_t *filteredCB, /* (i) The filtered CB memory (2:nd section) */ + size_t lMem, /* (i) Length of the CB memory */ + size_t lTarget, /* (i) Length of the target vector */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts, /* (o) Shift value of the energy */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size /* (i) Index to where energy values should be stored */ + ) { + int16_t *ppi, *ppo, *pp; + int32_t energy, tmp32; + + /* Compute the energy and store it in a vector. Also the + * corresponding shift values are stored. The energy values + * are reused in all three stages. */ + + /* Calculate the energy in the first block of 'lTarget' sampels. */ + ppi = CB+lMem-lTarget-1; + ppo = CB+lMem-1; + + pp=CB+lMem-lTarget; + energy = WebRtcSpl_DotProductWithScale( pp, pp, lTarget, scale); + + /* Normalize the energy and store the number of shifts */ + energyShifts[0] = (int16_t)WebRtcSpl_NormW32(energy); + tmp32 = energy << energyShifts[0]; + energyW16[0] = (int16_t)(tmp32 >> 16); + + /* Compute the energy of the rest of the cb memory + * by step wise adding and subtracting the next + * sample and the last sample respectively. */ + WebRtcIlbcfix_CbMemEnergyCalc(energy, range, ppi, ppo, energyW16, energyShifts, scale, 0); + + /* Next, precompute the energy values for the filtered cb section */ + energy=0; + pp=filteredCB+lMem-lTarget; + + energy = WebRtcSpl_DotProductWithScale( pp, pp, lTarget, scale); + + /* Normalize the energy and store the number of shifts */ + energyShifts[base_size] = (int16_t)WebRtcSpl_NormW32(energy); + tmp32 = energy << energyShifts[base_size]; + energyW16[base_size] = (int16_t)(tmp32 >> 16); + + ppi = filteredCB + lMem - 1 - lTarget; + ppo = filteredCB + lMem - 1; + + WebRtcIlbcfix_CbMemEnergyCalc(energy, range, ppi, ppo, energyW16, energyShifts, scale, base_size); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.h new file mode 100644 index 0000000000..17ec337dc6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergy.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_H_ + +#include +#include + +void WebRtcIlbcfix_CbMemEnergy( + size_t range, + int16_t* CB, /* (i) The CB memory (1:st section) */ + int16_t* filteredCB, /* (i) The filtered CB memory (2:nd section) */ + size_t lMem, /* (i) Length of the CB memory */ + size_t lTarget, /* (i) Length of the target vector */ + int16_t* energyW16, /* (o) Energy in the CB vectors */ + int16_t* energyShifts, /* (o) Shift value of the energy */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size /* (i) Index to where energy values should be stored */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c new file mode 100644 index 0000000000..0619bbe422 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergyAugmentation.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +void WebRtcIlbcfix_CbMemEnergyAugmentation( + int16_t *interpSamples, /* (i) The interpolated samples */ + int16_t *CBmem, /* (i) The CB memory */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size, /* (i) Index to where energy values should be stored */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts /* (o) Shift value of the energy */ + ){ + int32_t energy, tmp32; + int16_t *ppe, *pp, *interpSamplesPtr; + int16_t *CBmemPtr; + size_t lagcount; + int16_t *enPtr=&energyW16[base_size-20]; + int16_t *enShPtr=&energyShifts[base_size-20]; + int32_t nrjRecursive; + + CBmemPtr = CBmem+147; + interpSamplesPtr = interpSamples; + + /* Compute the energy for the first (low-5) noninterpolated samples */ + nrjRecursive = WebRtcSpl_DotProductWithScale( CBmemPtr-19, CBmemPtr-19, 15, scale); + ppe = CBmemPtr - 20; + + for (lagcount=20; lagcount<=39; lagcount++) { + + /* Update the energy recursively to save complexity */ + nrjRecursive += (*ppe * *ppe) >> scale; + ppe--; + energy = nrjRecursive; + + /* interpolation */ + energy += WebRtcSpl_DotProductWithScale(interpSamplesPtr, interpSamplesPtr, 4, scale); + interpSamplesPtr += 4; + + /* Compute energy for the remaining samples */ + pp = CBmemPtr - lagcount; + energy += WebRtcSpl_DotProductWithScale(pp, pp, SUBL-lagcount, scale); + + /* Normalize the energy and store the number of shifts */ + (*enShPtr) = (int16_t)WebRtcSpl_NormW32(energy); + tmp32 = energy << *enShPtr; + *enPtr = (int16_t)(tmp32 >> 16); + enShPtr++; + enPtr++; + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h new file mode 100644 index 0000000000..d7b7a0d97e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergyAugmentation.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_AUGMENTATION_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_AUGMENTATION_H_ + +#include +#include + +void WebRtcIlbcfix_CbMemEnergyAugmentation( + int16_t* interpSamples, /* (i) The interpolated samples */ + int16_t* CBmem, /* (i) The CB memory */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size, /* (i) Index to where energy values should be stored */ + int16_t* energyW16, /* (o) Energy in the CB vectors */ + int16_t* energyShifts /* (o) Shift value of the energy */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c new file mode 100644 index 0000000000..58c0c5fe6d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergyCalc.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/* Compute the energy of the rest of the cb memory + * by step wise adding and subtracting the next + * sample and the last sample respectively */ +void WebRtcIlbcfix_CbMemEnergyCalc( + int32_t energy, /* (i) input start energy */ + size_t range, /* (i) number of iterations */ + int16_t *ppi, /* (i) input pointer 1 */ + int16_t *ppo, /* (i) input pointer 2 */ + int16_t *energyW16, /* (o) Energy in the CB vectors */ + int16_t *energyShifts, /* (o) Shift value of the energy */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size /* (i) Index to where energy values should be stored */ + ) +{ + size_t j; + int16_t shft; + int32_t tmp; + int16_t *eSh_ptr; + int16_t *eW16_ptr; + + + eSh_ptr = &energyShifts[1+base_size]; + eW16_ptr = &energyW16[1+base_size]; + + for (j = 0; j + 1 < range; j++) { + + /* Calculate next energy by a +/- + operation on the edge samples */ + tmp = (*ppi) * (*ppi) - (*ppo) * (*ppo); + energy += tmp >> scale; + energy = WEBRTC_SPL_MAX(energy, 0); + + ppi--; + ppo--; + + /* Normalize the energy into a int16_t and store + the number of shifts */ + + shft = (int16_t)WebRtcSpl_NormW32(energy); + *eSh_ptr++ = shft; + + tmp = energy << shft; + *eW16_ptr++ = (int16_t)(tmp >> 16); + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h new file mode 100644 index 0000000000..1d1e8d62b9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbMemEnergyCalc.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_CALC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_MEM_ENERGY_CALC_H_ + +#include +#include + +void WebRtcIlbcfix_CbMemEnergyCalc( + int32_t energy, /* (i) input start energy */ + size_t range, /* (i) number of iterations */ + int16_t* ppi, /* (i) input pointer 1 */ + int16_t* ppo, /* (i) input pointer 2 */ + int16_t* energyW16, /* (o) Energy in the CB vectors */ + int16_t* energyShifts, /* (o) Shift value of the energy */ + int scale, /* (i) The scaling of all energy values */ + size_t base_size /* (i) Index to where energy values should be stored */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.c new file mode 100644 index 0000000000..24b5292354 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.c @@ -0,0 +1,405 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbSearch.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/cb_search.h" + +#include "modules/audio_coding/codecs/ilbc/augmented_cb_corr.h" +#include "modules/audio_coding/codecs/ilbc/cb_mem_energy.h" +#include "modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.h" +#include "modules/audio_coding/codecs/ilbc/cb_search_core.h" +#include "modules/audio_coding/codecs/ilbc/cb_update_best_index.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/create_augmented_vec.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/energy_inverse.h" +#include "modules/audio_coding/codecs/ilbc/filtered_cb_vecs.h" +#include "modules/audio_coding/codecs/ilbc/gain_quant.h" +#include "modules/audio_coding/codecs/ilbc/interpolate_samples.h" + +/*----------------------------------------------------------------* + * Search routine for codebook encoding and gain quantization. + *----------------------------------------------------------------*/ + +void WebRtcIlbcfix_CbSearch( + IlbcEncoder *iLBCenc_inst, + /* (i) the encoder state structure */ + int16_t *index, /* (o) Codebook indices */ + int16_t *gain_index, /* (o) Gain quantization indices */ + int16_t *intarget, /* (i) Target vector for encoding */ + int16_t *decResidual,/* (i) Decoded residual for codebook construction */ + size_t lMem, /* (i) Length of buffer */ + size_t lTarget, /* (i) Length of vector */ + int16_t *weightDenum,/* (i) weighting filter coefficients in Q12 */ + size_t block /* (i) the subblock number */ + ) { + size_t i, range; + int16_t ii, j, stage; + int16_t *pp; + int16_t tmp; + int scale; + int16_t bits, temp1, temp2; + size_t base_size; + int32_t codedEner, targetEner; + int16_t gains[CB_NSTAGES+1]; + int16_t *cb_vecPtr; + size_t indexOffset, sInd, eInd; + int32_t CritMax=0; + int16_t shTotMax=WEBRTC_SPL_WORD16_MIN; + size_t bestIndex=0; + int16_t bestGain=0; + size_t indexNew; + int16_t CritNewSh; + int32_t CritNew; + int32_t *cDotPtr; + size_t noOfZeros; + int16_t *gainPtr; + int32_t t32, tmpW32; + int16_t *WebRtcIlbcfix_kGainSq5_ptr; + /* Stack based */ + int16_t CBbuf[CB_MEML+LPC_FILTERORDER+CB_HALFFILTERLEN]; + int32_t cDot[128]; + int32_t Crit[128]; + int16_t targetVec[SUBL+LPC_FILTERORDER]; + int16_t cbvectors[CB_MEML + 1]; /* Adding one extra position for + Coverity warnings. */ + int16_t codedVec[SUBL]; + int16_t interpSamples[20*4]; + int16_t interpSamplesFilt[20*4]; + int16_t energyW16[CB_EXPAND*128]; + int16_t energyShifts[CB_EXPAND*128]; + int16_t *inverseEnergy=energyW16; /* Reuse memory */ + int16_t *inverseEnergyShifts=energyShifts; /* Reuse memory */ + int16_t *buf = &CBbuf[LPC_FILTERORDER]; + int16_t *target = &targetVec[LPC_FILTERORDER]; + int16_t *aug_vec = (int16_t*)cDot; /* length [SUBL], reuse memory */ + + /* Determine size of codebook sections */ + + base_size=lMem-lTarget+1; + if (lTarget==SUBL) { + base_size=lMem-19; + } + + /* weighting of the CB memory */ + noOfZeros=lMem-WebRtcIlbcfix_kFilterRange[block]; + WebRtcSpl_MemSetW16(&buf[-LPC_FILTERORDER], 0, noOfZeros+LPC_FILTERORDER); + WebRtcSpl_FilterARFastQ12( + decResidual+noOfZeros, buf+noOfZeros, + weightDenum, LPC_FILTERORDER+1, WebRtcIlbcfix_kFilterRange[block]); + + /* weighting of the target vector */ + WEBRTC_SPL_MEMCPY_W16(&target[-LPC_FILTERORDER], buf+noOfZeros+WebRtcIlbcfix_kFilterRange[block]-LPC_FILTERORDER, LPC_FILTERORDER); + WebRtcSpl_FilterARFastQ12( + intarget, target, + weightDenum, LPC_FILTERORDER+1, lTarget); + + /* Store target, towards the end codedVec is calculated as + the initial target minus the remaining target */ + WEBRTC_SPL_MEMCPY_W16(codedVec, target, lTarget); + + /* Find the highest absolute value to calculate proper + vector scale factor (so that it uses 12 bits) */ + temp1 = WebRtcSpl_MaxAbsValueW16(buf, lMem); + temp2 = WebRtcSpl_MaxAbsValueW16(target, lTarget); + + if ((temp1>0)&&(temp2>0)) { + temp1 = WEBRTC_SPL_MAX(temp1, temp2); + scale = WebRtcSpl_GetSizeInBits((uint32_t)(temp1 * temp1)); + } else { + /* temp1 or temp2 is negative (maximum was -32768) */ + scale = 30; + } + + /* Scale to so that a mul-add 40 times does not overflow */ + scale = scale - 25; + scale = WEBRTC_SPL_MAX(0, scale); + + /* Compute energy of the original target */ + targetEner = WebRtcSpl_DotProductWithScale(target, target, lTarget, scale); + + /* Prepare search over one more codebook section. This section + is created by filtering the original buffer with a filter. */ + WebRtcIlbcfix_FilteredCbVecs(cbvectors, buf, lMem, WebRtcIlbcfix_kFilterRange[block]); + + range = WebRtcIlbcfix_kSearchRange[block][0]; + + if(lTarget == SUBL) { + /* Create the interpolated samples and store them for use in all stages */ + + /* First section, non-filtered half of the cb */ + WebRtcIlbcfix_InterpolateSamples(interpSamples, buf, lMem); + + /* Second section, filtered half of the cb */ + WebRtcIlbcfix_InterpolateSamples(interpSamplesFilt, cbvectors, lMem); + + /* Compute the CB vectors' energies for the first cb section (non-filtered) */ + WebRtcIlbcfix_CbMemEnergyAugmentation(interpSamples, buf, + scale, 20, energyW16, energyShifts); + + /* Compute the CB vectors' energies for the second cb section (filtered cb) */ + WebRtcIlbcfix_CbMemEnergyAugmentation(interpSamplesFilt, cbvectors, scale, + base_size + 20, energyW16, + energyShifts); + + /* Compute the CB vectors' energies and store them in the vector + * energyW16. Also the corresponding shift values are stored. The + * energy values are used in all three stages. */ + WebRtcIlbcfix_CbMemEnergy(range, buf, cbvectors, lMem, + lTarget, energyW16+20, energyShifts+20, scale, base_size); + + } else { + /* Compute the CB vectors' energies and store them in the vector + * energyW16. Also the corresponding shift values are stored. The + * energy values are used in all three stages. */ + WebRtcIlbcfix_CbMemEnergy(range, buf, cbvectors, lMem, + lTarget, energyW16, energyShifts, scale, base_size); + + /* Set the energy positions 58-63 and 122-127 to zero + (otherwise they are uninitialized) */ + WebRtcSpl_MemSetW16(energyW16+range, 0, (base_size-range)); + WebRtcSpl_MemSetW16(energyW16+range+base_size, 0, (base_size-range)); + } + + /* Calculate Inverse Energy (energyW16 is already normalized + and will contain the inverse energy in Q29 after this call */ + WebRtcIlbcfix_EnergyInverse(energyW16, base_size*CB_EXPAND); + + /* The gain value computed in the previous stage is used + * as an upper limit to what the next stage gain value + * is allowed to be. In stage 0, 16384 (1.0 in Q14) is used as + * the upper limit. */ + gains[0] = 16384; + + for (stage=0; stage> 1) > bestIndex) ? + 0 : (bestIndex - (CB_RESRANGE >> 1)); + eInd=sInd+CB_RESRANGE; + if (eInd>=range) { + eInd=range-1; + sInd=eInd-CB_RESRANGE; + } + + range = WebRtcIlbcfix_kSearchRange[block][stage]; + + if (lTarget==SUBL) { + i=sInd; + if (sInd<20) { + WebRtcIlbcfix_AugmentedCbCorr(target, cbvectors + lMem, + interpSamplesFilt, cDot, sInd + 20, + WEBRTC_SPL_MIN(39, (eInd + 20)), scale); + i=20; + cDotPtr = &cDot[20 - sInd]; + } else { + cDotPtr = cDot; + } + + cb_vecPtr = cbvectors+lMem-20-i; + + /* Calculate the cross correlations (main part of the filtered CB) */ + WebRtcSpl_CrossCorrelation(cDotPtr, target, cb_vecPtr, lTarget, + eInd - i + 1, scale, -1); + + } else { + cDotPtr = cDot; + cb_vecPtr = cbvectors+lMem-lTarget-sInd; + + /* Calculate the cross correlations (main part of the filtered CB) */ + WebRtcSpl_CrossCorrelation(cDotPtr, target, cb_vecPtr, lTarget, + eInd - sInd + 1, scale, -1); + + } + + /* Adjust the search range for the augmented vectors */ + indexOffset=base_size+sInd; + + /* Search for best index in this part of the vector */ + WebRtcIlbcfix_CbSearchCore( + cDot, eInd-sInd+1, stage, inverseEnergy+indexOffset, + inverseEnergyShifts+indexOffset, Crit, + &indexNew, &CritNew, &CritNewSh); + + /* Update the global best index and the corresponding gain */ + WebRtcIlbcfix_CbUpdateBestIndex( + CritNew, CritNewSh, indexNew+indexOffset, cDot[indexNew], + inverseEnergy[indexNew+indexOffset], inverseEnergyShifts[indexNew+indexOffset], + &CritMax, &shTotMax, &bestIndex, &bestGain); + + index[stage] = (int16_t)bestIndex; + + + bestGain = WebRtcIlbcfix_GainQuant(bestGain, + (int16_t)WEBRTC_SPL_ABS_W16(gains[stage]), stage, &gain_index[stage]); + + /* Extract the best (according to measure) codebook vector + Also adjust the index, so that the augmented vectors are last. + Above these vectors were first... + */ + + if(lTarget==(STATE_LEN-iLBCenc_inst->state_short_len)) { + + if((size_t)index[stage]=20) { + /* Adjust index and extract vector */ + index[stage]-=20; + pp=buf+lMem-lTarget-index[stage]; + } else { + /* Adjust index and extract vector */ + index[stage]+=(int16_t)(base_size-20); + + WebRtcIlbcfix_CreateAugmentedVec(index[stage]-base_size+40, + buf+lMem, aug_vec); + pp = aug_vec; + + } + } else { + + if ((index[stage] - base_size) >= 20) { + /* Adjust index and extract vector */ + index[stage]-=20; + pp=cbvectors+lMem-lTarget- + index[stage]+base_size; + } else { + /* Adjust index and extract vector */ + index[stage]+=(int16_t)(base_size-20); + WebRtcIlbcfix_CreateAugmentedVec(index[stage]-2*base_size+40, + cbvectors+lMem, aug_vec); + pp = aug_vec; + } + } + } + + /* Subtract the best codebook vector, according + to measure, from the target vector */ + + WebRtcSpl_AddAffineVectorToVector(target, pp, (int16_t)(-bestGain), + (int32_t)8192, (int16_t)14, lTarget); + + /* record quantized gain */ + gains[stage+1] = bestGain; + + } /* end of Main Loop. for (stage=0;... */ + + /* Calculte the coded vector (original target - what's left) */ + for (i=0;i> 14); + + targetEner = (int16_t)WEBRTC_SPL_SHIFT_W32(targetEner, -bits) * tmp; + + tmpW32 = ((int32_t)(gains[1]-1))<<1; + + /* Pointer to the table that contains + gain_sq5TblFIX * gain_sq5TblFIX in Q14 */ + gainPtr=(int16_t*)WebRtcIlbcfix_kGainSq5Sq+gain_index[0]; + temp1 = (int16_t)WEBRTC_SPL_SHIFT_W32(codedEner, -bits); + + WebRtcIlbcfix_kGainSq5_ptr = (int16_t*)&WebRtcIlbcfix_kGainSq5[j]; + + /* targetEner and codedEner are in Q(-2*scale) */ + for (ii=gain_index[0];ii<32;ii++) { + + /* Change the index if + (codedEnergy*gainTbl[i]*gainTbl[i])<(targetEn*gain[0]*gain[0]) AND + gainTbl[i] < 2*gain[0] + */ + + t32 = temp1 * *gainPtr; + t32 = t32 - targetEner; + if (t32 < 0) { + if ((*WebRtcIlbcfix_kGainSq5_ptr) < tmpW32) { + j=ii; + WebRtcIlbcfix_kGainSq5_ptr = (int16_t*)&WebRtcIlbcfix_kGainSq5[ii]; + } + } + gainPtr++; + } + gain_index[0]=j; + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.h new file mode 100644 index 0000000000..84a52c7868 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbSearch.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_SEARCH_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_SEARCH_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +void WebRtcIlbcfix_CbSearch( + IlbcEncoder* iLBCenc_inst, + /* (i) the encoder state structure */ + int16_t* index, /* (o) Codebook indices */ + int16_t* gain_index, /* (o) Gain quantization indices */ + int16_t* intarget, /* (i) Target vector for encoding */ + int16_t* decResidual, /* (i) Decoded residual for codebook construction */ + size_t lMem, /* (i) Length of buffer */ + size_t lTarget, /* (i) Length of vector */ + int16_t* weightDenum, /* (i) weighting filter coefficients in Q12 */ + size_t block /* (i) the subblock number */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c new file mode 100644 index 0000000000..a75e5b0ab8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbSearchCore.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/cb_search_core.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +void WebRtcIlbcfix_CbSearchCore( + int32_t *cDot, /* (i) Cross Correlation */ + size_t range, /* (i) Search range */ + int16_t stage, /* (i) Stage of this search */ + int16_t *inverseEnergy, /* (i) Inversed energy */ + int16_t *inverseEnergyShift, /* (i) Shifts of inversed energy + with the offset 2*16-29 */ + int32_t *Crit, /* (o) The criteria */ + size_t *bestIndex, /* (o) Index that corresponds to + maximum criteria (in this + vector) */ + int32_t *bestCrit, /* (o) Value of critera for the + chosen index */ + int16_t *bestCritSh) /* (o) The domain of the chosen + criteria */ +{ + int32_t maxW32, tmp32; + int16_t max, sh, tmp16; + size_t i; + int32_t *cDotPtr; + int16_t cDotSqW16; + int16_t *inverseEnergyPtr; + int32_t *critPtr; + int16_t *inverseEnergyShiftPtr; + + /* Don't allow negative values for stage 0 */ + if (stage==0) { + cDotPtr=cDot; + for (i=0;i> 16); + cDotSqW16 = (int16_t)(((int32_t)(tmp16)*(tmp16))>>16); + + /* Calculate the criteria (cDot*cDot/energy) */ + *critPtr = cDotSqW16 * *inverseEnergyPtr; + + /* Extract the maximum shift value under the constraint + that the criteria is not zero */ + if ((*critPtr)!=0) { + max = WEBRTC_SPL_MAX((*inverseEnergyShiftPtr), max); + } + + inverseEnergyPtr++; + inverseEnergyShiftPtr++; + critPtr++; + cDotPtr++; + } + + /* If no max shifts still at initialization value, set shift to zero */ + if (max==WEBRTC_SPL_WORD16_MIN) { + max = 0; + } + + /* Modify the criterias, so that all of them use the same Q domain */ + critPtr=Crit; + inverseEnergyShiftPtr=inverseEnergyShift; + for (i=0;i31) */ + tmp16 = WEBRTC_SPL_MIN(16, max-(*inverseEnergyShiftPtr)); + + (*critPtr)=WEBRTC_SPL_SHIFT_W32((*critPtr),-tmp16); + critPtr++; + inverseEnergyShiftPtr++; + } + + /* Find the index of the best value */ + *bestIndex = WebRtcSpl_MaxIndexW32(Crit, range); + *bestCrit = Crit[*bestIndex]; + + /* Calculate total shifts of this criteria */ + *bestCritSh = 32 - 2*sh + max; + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.h new file mode 100644 index 0000000000..5da70e0988 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbSearchCore.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_SEARCH_CORE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_SEARCH_CORE_H_ + +#include +#include + +void WebRtcIlbcfix_CbSearchCore( + int32_t* cDot, /* (i) Cross Correlation */ + size_t range, /* (i) Search range */ + int16_t stage, /* (i) Stage of this search */ + int16_t* inverseEnergy, /* (i) Inversed energy */ + int16_t* inverseEnergyShift, /* (i) Shifts of inversed energy + with the offset 2*16-29 */ + int32_t* Crit, /* (o) The criteria */ + size_t* bestIndex, /* (o) Index that corresponds to + maximum criteria (in this + vector) */ + int32_t* bestCrit, /* (o) Value of critera for the + chosen index */ + int16_t* bestCritSh); /* (o) The domain of the chosen + criteria */ + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c new file mode 100644 index 0000000000..d6fa4d93d4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbUpdateBestIndex.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/cb_update_best_index.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +void WebRtcIlbcfix_CbUpdateBestIndex( + int32_t CritNew, /* (i) New Potentially best Criteria */ + int16_t CritNewSh, /* (i) Shift value of above Criteria */ + size_t IndexNew, /* (i) Index of new Criteria */ + int32_t cDotNew, /* (i) Cross dot of new index */ + int16_t invEnergyNew, /* (i) Inversed energy new index */ + int16_t energyShiftNew, /* (i) Energy shifts of new index */ + int32_t *CritMax, /* (i/o) Maximum Criteria (so far) */ + int16_t *shTotMax, /* (i/o) Shifts of maximum criteria */ + size_t *bestIndex, /* (i/o) Index that corresponds to + maximum criteria */ + int16_t *bestGain) /* (i/o) Gain in Q14 that corresponds + to maximum criteria */ +{ + int16_t shOld, shNew, tmp16; + int16_t scaleTmp; + int32_t gainW32; + + /* Normalize the new and old Criteria to the same domain */ + if (CritNewSh>(*shTotMax)) { + shOld=WEBRTC_SPL_MIN(31,CritNewSh-(*shTotMax)); + shNew=0; + } else { + shOld=0; + shNew=WEBRTC_SPL_MIN(31,(*shTotMax)-CritNewSh); + } + + /* Compare the two criterias. If the new one is better, + calculate the gain and store this index as the new best one + */ + + if ((CritNew >> shNew) > (*CritMax >> shOld)) { + + tmp16 = (int16_t)WebRtcSpl_NormW32(cDotNew); + tmp16 = 16 - tmp16; + + /* Calculate the gain in Q14 + Compensate for inverseEnergyshift in Q29 and that the energy + value was stored in a int16_t (shifted down 16 steps) + => 29-14+16 = 31 */ + + scaleTmp = -energyShiftNew-tmp16+31; + scaleTmp = WEBRTC_SPL_MIN(31, scaleTmp); + + gainW32 = ((int16_t)WEBRTC_SPL_SHIFT_W32(cDotNew, -tmp16) * invEnergyNew) >> + scaleTmp; + + /* Check if criteria satisfies Gain criteria (max 1.3) + if it is larger set the gain to 1.3 + (slightly different from FLP version) + */ + if (gainW32>21299) { + *bestGain=21299; + } else if (gainW32<-21299) { + *bestGain=-21299; + } else { + *bestGain=(int16_t)gainW32; + } + + *CritMax=CritNew; + *shTotMax=CritNewSh; + *bestIndex = IndexNew; + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.h new file mode 100644 index 0000000000..1a95d531e9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CbUpdateBestIndex.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_UPDATE_BEST_INDEX_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CB_UPDATE_BEST_INDEX_H_ + +#include +#include + +void WebRtcIlbcfix_CbUpdateBestIndex( + int32_t CritNew, /* (i) New Potentially best Criteria */ + int16_t CritNewSh, /* (i) Shift value of above Criteria */ + size_t IndexNew, /* (i) Index of new Criteria */ + int32_t cDotNew, /* (i) Cross dot of new index */ + int16_t invEnergyNew, /* (i) Inversed energy new index */ + int16_t energyShiftNew, /* (i) Energy shifts of new index */ + int32_t* CritMax, /* (i/o) Maximum Criteria (so far) */ + int16_t* shTotMax, /* (i/o) Shifts of maximum criteria */ + size_t* bestIndex, /* (i/o) Index that corresponds to + maximum criteria */ + int16_t* bestGain); /* (i/o) Gain in Q14 that corresponds + to maximum criteria */ + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.c new file mode 100644 index 0000000000..b4eee66219 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.c @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Chebyshev.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/chebyshev.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*------------------------------------------------------------------* + * Calculate the Chevyshev polynomial series + * F(w) = 2*exp(-j5w)*C(x) + * C(x) = (T_0(x) + f(1)T_1(x) + ... + f(4)T_1(x) + f(5)/2) + * T_i(x) is the i:th order Chebyshev polynomial + *------------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_Chebyshev( + /* (o) Result of C(x) */ + int16_t x, /* (i) Value to the Chevyshev polynomial */ + int16_t *f /* (i) The coefficients in the polynomial */ + ) { + int16_t b1_high, b1_low; /* Use the high, low format to increase the accuracy */ + int32_t b2; + int32_t tmp1W32; + int32_t tmp2W32; + int i; + + b2 = (int32_t)0x1000000; /* b2 = 1.0 (Q23) */ + /* Calculate b1 = 2*x + f[1] */ + tmp1W32 = (x << 10) + (f[1] << 14); + + for (i = 2; i < 5; i++) { + tmp2W32 = tmp1W32; + + /* Split b1 (in tmp1W32) into a high and low part */ + b1_high = (int16_t)(tmp1W32 >> 16); + b1_low = (int16_t)((tmp1W32 - ((int32_t)b1_high << 16)) >> 1); + + /* Calculate 2*x*b1-b2+f[i] */ + tmp1W32 = ((b1_high * x + ((b1_low * x) >> 15)) << 2) - b2 + (f[i] << 14); + + /* Update b2 for next round */ + b2 = tmp2W32; + } + + /* Split b1 (in tmp1W32) into a high and low part */ + b1_high = (int16_t)(tmp1W32 >> 16); + b1_low = (int16_t)((tmp1W32 - ((int32_t)b1_high << 16)) >> 1); + + /* tmp1W32 = x*b1 - b2 + f[i]/2 */ + tmp1W32 = ((b1_high * x) << 1) + (((b1_low * x) >> 15) << 1) - + b2 + (f[i] << 13); + + /* Handle overflows and set to maximum or minimum int16_t instead */ + if (tmp1W32>((int32_t)33553408)) { + return(WEBRTC_SPL_WORD16_MAX); + } else if (tmp1W32<((int32_t)-33554432)) { + return(WEBRTC_SPL_WORD16_MIN); + } else { + return (int16_t)(tmp1W32 >> 10); + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.h new file mode 100644 index 0000000000..7e7742c5cc --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Chebyshev.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CHEBYSHEV_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CHEBYSHEV_H_ + +#include +#include + +/*------------------------------------------------------------------* + * Calculate the Chevyshev polynomial series + * F(w) = 2*exp(-j5w)*C(x) + * C(x) = (T_0(x) + f(1)T_1(x) + ... + f(4)T_1(x) + f(5)/2) + * T_i(x) is the i:th order Chebyshev polynomial + *------------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_Chebyshev( + /* (o) Result of C(x) */ + int16_t x, /* (i) Value to the Chevyshev polynomial */ + int16_t* f /* (i) The coefficients in the polynomial */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.c new file mode 100644 index 0000000000..452bc78e3b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CompCorr.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/comp_corr.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Compute cross correlation and pitch gain for pitch prediction + * of last subframe at given lag. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_CompCorr( + int32_t *corr, /* (o) cross correlation */ + int32_t *ener, /* (o) energy */ + int16_t *buffer, /* (i) signal buffer */ + size_t lag, /* (i) pitch lag */ + size_t bLen, /* (i) length of buffer */ + size_t sRange, /* (i) correlation search length */ + int16_t scale /* (i) number of rightshifts to use */ + ){ + int16_t *w16ptr; + + w16ptr=&buffer[bLen-sRange-lag]; + + /* Calculate correlation and energy */ + (*corr)=WebRtcSpl_DotProductWithScale(&buffer[bLen-sRange], w16ptr, sRange, scale); + (*ener)=WebRtcSpl_DotProductWithScale(w16ptr, w16ptr, sRange, scale); + + /* For zero energy set the energy to 0 in order to avoid potential + problems for coming divisions */ + if (*ener == 0) { + *corr = 0; + *ener = 1; + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.h new file mode 100644 index 0000000000..010c6a1ce5 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CompCorr.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_COMP_CORR_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_COMP_CORR_H_ + +#include +#include + +/*----------------------------------------------------------------* + * Compute cross correlation and pitch gain for pitch prediction + * of last subframe at given lag. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_CompCorr(int32_t* corr, /* (o) cross correlation */ + int32_t* ener, /* (o) energy */ + int16_t* buffer, /* (i) signal buffer */ + size_t lag, /* (i) pitch lag */ + size_t bLen, /* (i) length of buffer */ + size_t sRange, /* (i) correlation search length */ + int16_t scale /* (i) number of rightshifts to use */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/complexityMeasures.m b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/complexityMeasures.m new file mode 100644 index 0000000000..4bda83622f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/complexityMeasures.m @@ -0,0 +1,57 @@ +% % Copyright(c) 2011 The WebRTC project authors.All Rights Reserved.% + % Use of this source code is governed by a BSD + - + style license % that can be found in the LICENSE file in the root of the source + % tree.An additional intellectual property rights grant can be found + % in the file PATENTS.All contributing project authors may + % be found in the AUTHORS file in the root of the source tree.% + + clear; +pack; +% +% Enter the path to YOUR executable and remember to define the perprocessor +% variable PRINT_MIPS te get the instructions printed to the screen. +% +command = '!iLBCtest.exe 30 speechAndBGnoise.pcm out1.bit out1.pcm tlm10_30ms.dat'; +cout=' > st.txt'; %saves to matlab variable 'st' +eval(strcat(command,cout)); +if(length(cout)>3) + load st.txt +else + disp('No cout file to load') +end + +% initialize vector to zero +index = find(st(1:end,1)==-1); +indexnonzero = find(st(1:end,1)>0); +frames = length(index)-indexnonzero(1)+1; +start = indexnonzero(1) - 1; +functionOrder=max(st(:,2)); +new=zeros(frames,functionOrder); + +for i = 1:frames, + for j = index(start-1+i)+1:(index(start+i)-1), + new(i,st(j,2)) = new(i,st(j,2)) + st(j,1); + end +end + +result=zeros(functionOrder,3); +for i=1:functionOrder + nonzeroelements = find(new(1:end,i)>0); + result(i,1)=i; + + % Compute each function's mean complexity + % result(i,2)=(sum(new(nonzeroelements,i))/(length(nonzeroelements)*0.03))/1000000; + + % Compute each function's maximum complexity in encoding + % and decoding respectively and then add it together: + % result(i,3)=(max(new(1:end,i))/0.03)/1000000; + result(i,3)=(max(new(1:size(new,1)/2,i))/0.03)/1000000 + (max(new(size(new,1)/2+1:end,i))/0.03)/1000000; +end + +result + +% Compute maximum complexity for a single frame (enc/dec separately and together) +maxEncComplexityInAFrame = (max(sum(new(1:size(new,1)/2,:),2))/0.03)/1000000 +maxDecComplexityInAFrame = (max(sum(new(size(new,1)/2+1:end,:),2))/0.03)/1000000 +totalComplexity = maxEncComplexityInAFrame + maxDecComplexityInAFrame diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.c new file mode 100644 index 0000000000..22f2acb330 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.c @@ -0,0 +1,667 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + constants.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/constants.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/* HP Filters {b[0] b[1] b[2] -a[1] -a[2]} */ + +const int16_t WebRtcIlbcfix_kHpInCoefs[5] = {3798, -7596, 3798, 7807, -3733}; +const int16_t WebRtcIlbcfix_kHpOutCoefs[5] = {3849, -7699, 3849, 7918, -3833}; + +/* Window in Q11 to window the energies of the 5 choises (3 for 20ms) in the choise for + the 80 sample start state +*/ +const int16_t WebRtcIlbcfix_kStartSequenceEnrgWin[NSUB_MAX-1]= { + 1638, 1843, 2048, 1843, 1638 +}; + +/* LP Filter coeffs used for downsampling */ +const int16_t WebRtcIlbcfix_kLpFiltCoefs[FILTERORDER_DS_PLUS1]= { + -273, 512, 1297, 1696, 1297, 512, -273 +}; + +/* Constants used in the LPC calculations */ + +/* Hanning LPC window (in Q15) */ +const int16_t WebRtcIlbcfix_kLpcWin[BLOCKL_MAX] = { + 6, 22, 50, 89, 139, 200, 272, 355, 449, 554, 669, 795, + 932, 1079, 1237, 1405, 1583, 1771, 1969, 2177, 2395, 2622, 2858, 3104, + 3359, 3622, 3894, 4175, 4464, 4761, 5066, 5379, 5699, 6026, 6361, 6702, + 7050, 7404, 7764, 8130, 8502, 8879, 9262, 9649, 10040, 10436, 10836, 11240, + 11647, 12058, 12471, 12887, 13306, 13726, 14148, 14572, 14997, 15423, 15850, 16277, + 16704, 17131, 17558, 17983, 18408, 18831, 19252, 19672, 20089, 20504, 20916, 21325, + 21730, 22132, 22530, 22924, 23314, 23698, 24078, 24452, 24821, 25185, 25542, 25893, + 26238, 26575, 26906, 27230, 27547, 27855, 28156, 28450, 28734, 29011, 29279, 29538, + 29788, 30029, 30261, 30483, 30696, 30899, 31092, 31275, 31448, 31611, 31764, 31906, + 32037, 32158, 32268, 32367, 32456, 32533, 32600, 32655, 32700, 32733, 32755, 32767, + 32767, 32755, 32733, 32700, 32655, 32600, 32533, 32456, 32367, 32268, 32158, 32037, + 31906, 31764, 31611, 31448, 31275, 31092, 30899, 30696, 30483, 30261, 30029, 29788, + 29538, 29279, 29011, 28734, 28450, 28156, 27855, 27547, 27230, 26906, 26575, 26238, + 25893, 25542, 25185, 24821, 24452, 24078, 23698, 23314, 22924, 22530, 22132, 21730, + 21325, 20916, 20504, 20089, 19672, 19252, 18831, 18408, 17983, 17558, 17131, 16704, + 16277, 15850, 15423, 14997, 14572, 14148, 13726, 13306, 12887, 12471, 12058, 11647, + 11240, 10836, 10436, 10040, 9649, 9262, 8879, 8502, 8130, 7764, 7404, 7050, + 6702, 6361, 6026, 5699, 5379, 5066, 4761, 4464, 4175, 3894, 3622, 3359, + 3104, 2858, 2622, 2395, 2177, 1969, 1771, 1583, 1405, 1237, 1079, 932, + 795, 669, 554, 449, 355, 272, 200, 139, 89, 50, 22, 6 +}; + +/* Asymmetric LPC window (in Q15)*/ +const int16_t WebRtcIlbcfix_kLpcAsymWin[BLOCKL_MAX] = { + 2, 7, 15, 27, 42, 60, 81, 106, 135, 166, 201, 239, + 280, 325, 373, 424, 478, 536, 597, 661, 728, 798, 872, 949, + 1028, 1111, 1197, 1287, 1379, 1474, 1572, 1674, 1778, 1885, 1995, 2108, + 2224, 2343, 2465, 2589, 2717, 2847, 2980, 3115, 3254, 3395, 3538, 3684, + 3833, 3984, 4138, 4295, 4453, 4615, 4778, 4944, 5112, 5283, 5456, 5631, + 5808, 5987, 6169, 6352, 6538, 6725, 6915, 7106, 7300, 7495, 7692, 7891, + 8091, 8293, 8497, 8702, 8909, 9118, 9328, 9539, 9752, 9966, 10182, 10398, + 10616, 10835, 11055, 11277, 11499, 11722, 11947, 12172, 12398, 12625, 12852, 13080, + 13309, 13539, 13769, 14000, 14231, 14463, 14695, 14927, 15160, 15393, 15626, 15859, + 16092, 16326, 16559, 16792, 17026, 17259, 17492, 17725, 17957, 18189, 18421, 18653, + 18884, 19114, 19344, 19573, 19802, 20030, 20257, 20483, 20709, 20934, 21157, 21380, + 21602, 21823, 22042, 22261, 22478, 22694, 22909, 23123, 23335, 23545, 23755, 23962, + 24168, 24373, 24576, 24777, 24977, 25175, 25371, 25565, 25758, 25948, 26137, 26323, + 26508, 26690, 26871, 27049, 27225, 27399, 27571, 27740, 27907, 28072, 28234, 28394, + 28552, 28707, 28860, 29010, 29157, 29302, 29444, 29584, 29721, 29855, 29987, 30115, + 30241, 30364, 30485, 30602, 30717, 30828, 30937, 31043, 31145, 31245, 31342, 31436, + 31526, 31614, 31699, 31780, 31858, 31933, 32005, 32074, 32140, 32202, 32261, 32317, + 32370, 32420, 32466, 32509, 32549, 32585, 32618, 32648, 32675, 32698, 32718, 32734, + 32748, 32758, 32764, 32767, 32767, 32667, 32365, 31863, 31164, 30274, 29197, 27939, + 26510, 24917, 23170, 21281, 19261, 17121, 14876, 12540, 10126, 7650, 5126, 2571 +}; + +/* Lag window for LPC (Q31) */ +const int32_t WebRtcIlbcfix_kLpcLagWin[LPC_FILTERORDER + 1]={ + 2147483647, 2144885453, 2137754373, 2125918626, 2109459810, + 2088483140, 2063130336, 2033564590, 1999977009, 1962580174, + 1921610283}; + +/* WebRtcIlbcfix_kLpcChirpSyntDenum vector in Q15 corresponding + * floating point vector {1 0.9025 0.9025^2 0.9025^3 ...} + */ +const int16_t WebRtcIlbcfix_kLpcChirpSyntDenum[LPC_FILTERORDER + 1] = { + 32767, 29573, 26690, 24087, + 21739, 19619, 17707, 15980, + 14422, 13016, 11747}; + +/* WebRtcIlbcfix_kLpcChirpWeightDenum in Q15 corresponding to + * floating point vector {1 0.4222 0.4222^2... } + */ +const int16_t WebRtcIlbcfix_kLpcChirpWeightDenum[LPC_FILTERORDER + 1] = { + 32767, 13835, 5841, 2466, 1041, 440, + 186, 78, 33, 14, 6}; + +/* LSF quantization Q13 domain */ +const int16_t WebRtcIlbcfix_kLsfCb[64 * 3 + 128 * 3 + 128 * 4] = { + 1273, 2238, 3696, + 3199, 5309, 8209, + 3606, 5671, 7829, + 2815, 5262, 8778, + 2608, 4027, 5493, + 1582, 3076, 5945, + 2983, 4181, 5396, + 2437, 4322, 6902, + 1861, 2998, 4613, + 2007, 3250, 5214, + 1388, 2459, 4262, + 2563, 3805, 5269, + 2036, 3522, 5129, + 1935, 4025, 6694, + 2744, 5121, 7338, + 2810, 4248, 5723, + 3054, 5405, 7745, + 1449, 2593, 4763, + 3411, 5128, 6596, + 2484, 4659, 7496, + 1668, 2879, 4818, + 1812, 3072, 5036, + 1638, 2649, 3900, + 2464, 3550, 4644, + 1853, 2900, 4158, + 2458, 4163, 5830, + 2556, 4036, 6254, + 2703, 4432, 6519, + 3062, 4953, 7609, + 1725, 3703, 6187, + 2221, 3877, 5427, + 2339, 3579, 5197, + 2021, 4633, 7037, + 2216, 3328, 4535, + 2961, 4739, 6667, + 2807, 3955, 5099, + 2788, 4501, 6088, + 1642, 2755, 4431, + 3341, 5282, 7333, + 2414, 3726, 5727, + 1582, 2822, 5269, + 2259, 3447, 4905, + 3117, 4986, 7054, + 1825, 3491, 5542, + 3338, 5736, 8627, + 1789, 3090, 5488, + 2566, 3720, 4923, + 2846, 4682, 7161, + 1950, 3321, 5976, + 1834, 3383, 6734, + 3238, 4769, 6094, + 2031, 3978, 5903, + 1877, 4068, 7436, + 2131, 4644, 8296, + 2764, 5010, 8013, + 2194, 3667, 6302, + 2053, 3127, 4342, + 3523, 6595, 10010, + 3134, 4457, 5748, + 3142, 5819, 9414, + 2223, 4334, 6353, + 2022, 3224, 4822, + 2186, 3458, 5544, + 2552, 4757, 6870, + 10905, 12917, 14578, + 9503, 11485, 14485, + 9518, 12494, 14052, + 6222, 7487, 9174, + 7759, 9186, 10506, + 8315, 12755, 14786, + 9609, 11486, 13866, + 8909, 12077, 13643, + 7369, 9054, 11520, + 9408, 12163, 14715, + 6436, 9911, 12843, + 7109, 9556, 11884, + 7557, 10075, 11640, + 6482, 9202, 11547, + 6463, 7914, 10980, + 8611, 10427, 12752, + 7101, 9676, 12606, + 7428, 11252, 13172, + 10197, 12955, 15842, + 7487, 10955, 12613, + 5575, 7858, 13621, + 7268, 11719, 14752, + 7476, 11744, 13795, + 7049, 8686, 11922, + 8234, 11314, 13983, + 6560, 11173, 14984, + 6405, 9211, 12337, + 8222, 12054, 13801, + 8039, 10728, 13255, + 10066, 12733, 14389, + 6016, 7338, 10040, + 6896, 8648, 10234, + 7538, 9170, 12175, + 7327, 12608, 14983, + 10516, 12643, 15223, + 5538, 7644, 12213, + 6728, 12221, 14253, + 7563, 9377, 12948, + 8661, 11023, 13401, + 7280, 8806, 11085, + 7723, 9793, 12333, + 12225, 14648, 16709, + 8768, 13389, 15245, + 10267, 12197, 13812, + 5301, 7078, 11484, + 7100, 10280, 11906, + 8716, 12555, 14183, + 9567, 12464, 15434, + 7832, 12305, 14300, + 7608, 10556, 12121, + 8913, 11311, 12868, + 7414, 9722, 11239, + 8666, 11641, 13250, + 9079, 10752, 12300, + 8024, 11608, 13306, + 10453, 13607, 16449, + 8135, 9573, 10909, + 6375, 7741, 10125, + 10025, 12217, 14874, + 6985, 11063, 14109, + 9296, 13051, 14642, + 8613, 10975, 12542, + 6583, 10414, 13534, + 6191, 9368, 13430, + 5742, 6859, 9260, + 7723, 9813, 13679, + 8137, 11291, 12833, + 6562, 8973, 10641, + 6062, 8462, 11335, + 6928, 8784, 12647, + 7501, 8784, 10031, + 8372, 10045, 12135, + 8191, 9864, 12746, + 5917, 7487, 10979, + 5516, 6848, 10318, + 6819, 9899, 11421, + 7882, 12912, 15670, + 9558, 11230, 12753, + 7752, 9327, 11472, + 8479, 9980, 11358, + 11418, 14072, 16386, + 7968, 10330, 14423, + 8423, 10555, 12162, + 6337, 10306, 14391, + 8850, 10879, 14276, + 6750, 11885, 15710, + 7037, 8328, 9764, + 6914, 9266, 13476, + 9746, 13949, 15519, + 11032, 14444, 16925, + 8032, 10271, 11810, + 10962, 13451, 15833, + 10021, 11667, 13324, + 6273, 8226, 12936, + 8543, 10397, 13496, + 7936, 10302, 12745, + 6769, 8138, 10446, + 6081, 7786, 11719, + 8637, 11795, 14975, + 8790, 10336, 11812, + 7040, 8490, 10771, + 7338, 10381, 13153, + 6598, 7888, 9358, + 6518, 8237, 12030, + 9055, 10763, 12983, + 6490, 10009, 12007, + 9589, 12023, 13632, + 6867, 9447, 10995, + 7930, 9816, 11397, + 10241, 13300, 14939, + 5830, 8670, 12387, + 9870, 11915, 14247, + 9318, 11647, 13272, + 6721, 10836, 12929, + 6543, 8233, 9944, + 8034, 10854, 12394, + 9112, 11787, 14218, + 9302, 11114, 13400, + 9022, 11366, 13816, + 6962, 10461, 12480, + 11288, 13333, 15222, + 7249, 8974, 10547, + 10566, 12336, 14390, + 6697, 11339, 13521, + 11851, 13944, 15826, + 6847, 8381, 11349, + 7509, 9331, 10939, + 8029, 9618, 11909, + 13973, 17644, 19647, 22474, + 14722, 16522, 20035, 22134, + 16305, 18179, 21106, 23048, + 15150, 17948, 21394, 23225, + 13582, 15191, 17687, 22333, + 11778, 15546, 18458, 21753, + 16619, 18410, 20827, 23559, + 14229, 15746, 17907, 22474, + 12465, 15327, 20700, 22831, + 15085, 16799, 20182, 23410, + 13026, 16935, 19890, 22892, + 14310, 16854, 19007, 22944, + 14210, 15897, 18891, 23154, + 14633, 18059, 20132, 22899, + 15246, 17781, 19780, 22640, + 16396, 18904, 20912, 23035, + 14618, 17401, 19510, 21672, + 15473, 17497, 19813, 23439, + 18851, 20736, 22323, 23864, + 15055, 16804, 18530, 20916, + 16490, 18196, 19990, 21939, + 11711, 15223, 21154, 23312, + 13294, 15546, 19393, 21472, + 12956, 16060, 20610, 22417, + 11628, 15843, 19617, 22501, + 14106, 16872, 19839, 22689, + 15655, 18192, 20161, 22452, + 12953, 15244, 20619, 23549, + 15322, 17193, 19926, 21762, + 16873, 18676, 20444, 22359, + 14874, 17871, 20083, 21959, + 11534, 14486, 19194, 21857, + 17766, 19617, 21338, 23178, + 13404, 15284, 19080, 23136, + 15392, 17527, 19470, 21953, + 14462, 16153, 17985, 21192, + 17734, 19750, 21903, 23783, + 16973, 19096, 21675, 23815, + 16597, 18936, 21257, 23461, + 15966, 17865, 20602, 22920, + 15416, 17456, 20301, 22972, + 18335, 20093, 21732, 23497, + 15548, 17217, 20679, 23594, + 15208, 16995, 20816, 22870, + 13890, 18015, 20531, 22468, + 13211, 15377, 19951, 22388, + 12852, 14635, 17978, 22680, + 16002, 17732, 20373, 23544, + 11373, 14134, 19534, 22707, + 17329, 19151, 21241, 23462, + 15612, 17296, 19362, 22850, + 15422, 19104, 21285, 23164, + 13792, 17111, 19349, 21370, + 15352, 17876, 20776, 22667, + 15253, 16961, 18921, 22123, + 14108, 17264, 20294, 23246, + 15785, 17897, 20010, 21822, + 17399, 19147, 20915, 22753, + 13010, 15659, 18127, 20840, + 16826, 19422, 22218, 24084, + 18108, 20641, 22695, 24237, + 18018, 20273, 22268, 23920, + 16057, 17821, 21365, 23665, + 16005, 17901, 19892, 23016, + 13232, 16683, 21107, 23221, + 13280, 16615, 19915, 21829, + 14950, 18575, 20599, 22511, + 16337, 18261, 20277, 23216, + 14306, 16477, 21203, 23158, + 12803, 17498, 20248, 22014, + 14327, 17068, 20160, 22006, + 14402, 17461, 21599, 23688, + 16968, 18834, 20896, 23055, + 15070, 17157, 20451, 22315, + 15419, 17107, 21601, 23946, + 16039, 17639, 19533, 21424, + 16326, 19261, 21745, 23673, + 16489, 18534, 21658, 23782, + 16594, 18471, 20549, 22807, + 18973, 21212, 22890, 24278, + 14264, 18674, 21123, 23071, + 15117, 16841, 19239, 23118, + 13762, 15782, 20478, 23230, + 14111, 15949, 20058, 22354, + 14990, 16738, 21139, 23492, + 13735, 16971, 19026, 22158, + 14676, 17314, 20232, 22807, + 16196, 18146, 20459, 22339, + 14747, 17258, 19315, 22437, + 14973, 17778, 20692, 23367, + 15715, 17472, 20385, 22349, + 15702, 18228, 20829, 23410, + 14428, 16188, 20541, 23630, + 16824, 19394, 21365, 23246, + 13069, 16392, 18900, 21121, + 12047, 16640, 19463, 21689, + 14757, 17433, 19659, 23125, + 15185, 16930, 19900, 22540, + 16026, 17725, 19618, 22399, + 16086, 18643, 21179, 23472, + 15462, 17248, 19102, 21196, + 17368, 20016, 22396, 24096, + 12340, 14475, 19665, 23362, + 13636, 16229, 19462, 22728, + 14096, 16211, 19591, 21635, + 12152, 14867, 19943, 22301, + 14492, 17503, 21002, 22728, + 14834, 16788, 19447, 21411, + 14650, 16433, 19326, 22308, + 14624, 16328, 19659, 23204, + 13888, 16572, 20665, 22488, + 12977, 16102, 18841, 22246, + 15523, 18431, 21757, 23738, + 14095, 16349, 18837, 20947, + 13266, 17809, 21088, 22839, + 15427, 18190, 20270, 23143, + 11859, 16753, 20935, 22486, + 12310, 17667, 21736, 23319, + 14021, 15926, 18702, 22002, + 12286, 15299, 19178, 21126, + 15703, 17491, 21039, 23151, + 12272, 14018, 18213, 22570, + 14817, 16364, 18485, 22598, + 17109, 19683, 21851, 23677, + 12657, 14903, 19039, 22061, + 14713, 16487, 20527, 22814, + 14635, 16726, 18763, 21715, + 15878, 18550, 20718, 22906 +}; + +const int16_t WebRtcIlbcfix_kLsfDimCb[LSF_NSPLIT] = {3, 3, 4}; +const int16_t WebRtcIlbcfix_kLsfSizeCb[LSF_NSPLIT] = {64,128,128}; + +const int16_t WebRtcIlbcfix_kLsfMean[LPC_FILTERORDER] = { + 2308, 3652, 5434, 7885, + 10255, 12559, 15160, 17513, + 20328, 22752}; + +const int16_t WebRtcIlbcfix_kLspMean[LPC_FILTERORDER] = { + 31476, 29565, 25819, 18725, 10276, + 1236, -9049, -17600, -25884, -30618 +}; + +/* Q14 */ +const int16_t WebRtcIlbcfix_kLsfWeight20ms[4] = {12288, 8192, 4096, 0}; +const int16_t WebRtcIlbcfix_kLsfWeight30ms[6] = {8192, 16384, 10923, 5461, 0, 0}; + +/* + cos(x) in Q15 + WebRtcIlbcfix_kCos[i] = cos(pi*i/64.0) + used in WebRtcIlbcfix_Lsp2Lsf() +*/ + +const int16_t WebRtcIlbcfix_kCos[64] = { + 32767, 32729, 32610, 32413, 32138, 31786, 31357, 30853, + 30274, 29622, 28899, 28106, 27246, 26320, 25330, 24279, + 23170, 22006, 20788, 19520, 18205, 16846, 15447, 14010, + 12540, 11039, 9512, 7962, 6393, 4808, 3212, 1608, + 0, -1608, -3212, -4808, -6393, -7962, -9512, -11039, + -12540, -14010, -15447, -16846, -18205, -19520, -20788, -22006, + -23170, -24279, -25330, -26320, -27246, -28106, -28899, -29622, + -30274, -30853, -31357, -31786, -32138, -32413, -32610, -32729 +}; + +/* + Derivative in Q19, used to interpolate between the + WebRtcIlbcfix_kCos[] values to get a more exact y = cos(x) +*/ +const int16_t WebRtcIlbcfix_kCosDerivative[64] = { + -632, -1893, -3150, -4399, -5638, -6863, -8072, -9261, + -10428, -11570, -12684, -13767, -14817, -15832, -16808, -17744, + -18637, -19486, -20287, -21039, -21741, -22390, -22986, -23526, + -24009, -24435, -24801, -25108, -25354, -25540, -25664, -25726, + -25726, -25664, -25540, -25354, -25108, -24801, -24435, -24009, + -23526, -22986, -22390, -21741, -21039, -20287, -19486, -18637, + -17744, -16808, -15832, -14817, -13767, -12684, -11570, -10428, + -9261, -8072, -6863, -5638, -4399, -3150, -1893, -632}; + +/* + Table in Q15, used for a2lsf conversion + WebRtcIlbcfix_kCosGrid[i] = cos((2*pi*i)/(float)(2*COS_GRID_POINTS)); +*/ + +const int16_t WebRtcIlbcfix_kCosGrid[COS_GRID_POINTS + 1] = { + 32760, 32723, 32588, 32364, 32051, 31651, 31164, 30591, + 29935, 29196, 28377, 27481, 26509, 25465, 24351, 23170, + 21926, 20621, 19260, 17846, 16384, 14876, 13327, 11743, + 10125, 8480, 6812, 5126, 3425, 1714, 0, -1714, -3425, + -5126, -6812, -8480, -10125, -11743, -13327, -14876, + -16384, -17846, -19260, -20621, -21926, -23170, -24351, + -25465, -26509, -27481, -28377, -29196, -29935, -30591, + -31164, -31651, -32051, -32364, -32588, -32723, -32760 +}; + +/* + Derivative of y = acos(x) in Q12 + used in WebRtcIlbcfix_Lsp2Lsf() +*/ + +const int16_t WebRtcIlbcfix_kAcosDerivative[64] = { + -26887, -8812, -5323, -3813, -2979, -2444, -2081, -1811, + -1608, -1450, -1322, -1219, -1132, -1059, -998, -946, + -901, -861, -827, -797, -772, -750, -730, -713, + -699, -687, -677, -668, -662, -657, -654, -652, + -652, -654, -657, -662, -668, -677, -687, -699, + -713, -730, -750, -772, -797, -827, -861, -901, + -946, -998, -1059, -1132, -1219, -1322, -1450, -1608, + -1811, -2081, -2444, -2979, -3813, -5323, -8812, -26887 +}; + + +/* Tables for quantization of start state */ + +/* State quantization tables */ +const int16_t WebRtcIlbcfix_kStateSq3[8] = { /* Values in Q13 */ + -30473, -17838, -9257, -2537, + 3639, 10893, 19958, 32636 +}; + +/* This table defines the limits for the selection of the freqg + less or equal than value 0 => index = 0 + less or equal than value k => index = k +*/ +const int32_t WebRtcIlbcfix_kChooseFrgQuant[64] = { + 118, 163, 222, 305, 425, 604, + 851, 1174, 1617, 2222, 3080, 4191, + 5525, 7215, 9193, 11540, 14397, 17604, + 21204, 25209, 29863, 35720, 42531, 50375, + 59162, 68845, 80108, 93754, 110326, 129488, + 150654, 174328, 201962, 233195, 267843, 308239, + 354503, 405988, 464251, 531550, 608652, 697516, + 802526, 928793, 1080145, 1258120, 1481106, 1760881, + 2111111, 2546619, 3078825, 3748642, 4563142, 5573115, + 6887601, 8582108, 10797296, 14014513, 18625760, 25529599, + 37302935, 58819185, 109782723, WEBRTC_SPL_WORD32_MAX +}; + +const int16_t WebRtcIlbcfix_kScale[64] = { + /* Values in Q16 */ + 29485, 25003, 21345, 18316, 15578, 13128, 10973, 9310, 7955, + 6762, 5789, 4877, 4255, 3699, 3258, 2904, 2595, 2328, + 2123, 1932, 1785, 1631, 1493, 1370, 1260, 1167, 1083, + /* Values in Q21 */ + 32081, 29611, 27262, 25229, 23432, 21803, 20226, 18883, 17609, + 16408, 15311, 14327, 13390, 12513, 11693, 10919, 10163, 9435, + 8739, 8100, 7424, 6813, 6192, 5648, 5122, 4639, 4207, 3798, + 3404, 3048, 2706, 2348, 2036, 1713, 1393, 1087, 747 +}; + +/*frgq in fixpoint, but already computed like this: + for(i=0; i<64; i++){ + a = (pow(10,frgq[i])/4.5); + WebRtcIlbcfix_kFrgQuantMod[i] = round(a); + } + + Value 0 :36 in Q8 + 37:58 in Q5 + 59:63 in Q3 +*/ +const int16_t WebRtcIlbcfix_kFrgQuantMod[64] = { + /* First 37 values in Q8 */ + 569, 671, 786, 916, 1077, 1278, + 1529, 1802, 2109, 2481, 2898, 3440, + 3943, 4535, 5149, 5778, 6464, 7208, + 7904, 8682, 9397, 10285, 11240, 12246, + 13313, 14382, 15492, 16735, 18131, 19693, + 21280, 22912, 24624, 26544, 28432, 30488, + 32720, + /* 22 values in Q5 */ + 4383, 4684, 5012, 5363, 5739, 6146, + 6603, 7113, 7679, 8285, 9040, 9850, + 10838, 11882, 13103, 14467, 15950, 17669, + 19712, 22016, 24800, 28576, + /* 5 values in Q3 */ + 8240, 9792, 12040, 15440, 22472 +}; + +/* Constants for codebook search and creation */ + +/* Expansion filter to get additional cb section. + * Q12 and reversed compared to flp + */ +const int16_t WebRtcIlbcfix_kCbFiltersRev[CB_FILTERLEN]={ + -140, 446, -755, 3302, 2922, -590, 343, -138}; + +/* Weighting coefficients for short lags. + * [0.2 0.4 0.6 0.8] in Q15 */ +const int16_t WebRtcIlbcfix_kAlpha[4]={ + 6554, 13107, 19661, 26214}; + +/* Ranges for search and filters at different subframes */ + +const size_t WebRtcIlbcfix_kSearchRange[5][CB_NSTAGES]={ + {58,58,58}, {108,44,44}, {108,108,108}, {108,108,108}, {108,108,108}}; + +const size_t WebRtcIlbcfix_kFilterRange[5]={63, 85, 125, 147, 147}; + +/* Gain Quantization for the codebook gains of the 3 stages */ + +/* Q14 (one extra value (max int16_t) to simplify for the search) */ +const int16_t WebRtcIlbcfix_kGainSq3[9]={ + -16384, -10813, -5407, 0, 4096, 8192, + 12288, 16384, 32767}; + +/* Q14 (one extra value (max int16_t) to simplify for the search) */ +const int16_t WebRtcIlbcfix_kGainSq4[17]={ + -17203, -14746, -12288, -9830, -7373, -4915, + -2458, 0, 2458, 4915, 7373, 9830, + 12288, 14746, 17203, 19661, 32767}; + +/* Q14 (one extra value (max int16_t) to simplify for the search) */ +const int16_t WebRtcIlbcfix_kGainSq5[33]={ + 614, 1229, 1843, 2458, 3072, 3686, + 4301, 4915, 5530, 6144, 6758, 7373, + 7987, 8602, 9216, 9830, 10445, 11059, + 11674, 12288, 12902, 13517, 14131, 14746, + 15360, 15974, 16589, 17203, 17818, 18432, + 19046, 19661, 32767}; + +/* Q14 gain_sq5Tbl squared in Q14 */ +const int16_t WebRtcIlbcfix_kGainSq5Sq[32] = { + 23, 92, 207, 368, 576, 829, + 1129, 1474, 1866, 2304, 2787, 3317, + 3893, 4516, 5184, 5897, 6658, 7464, + 8318, 9216, 10160, 11151, 12187, 13271, + 14400, 15574, 16796, 18062, 19377, 20736, + 22140, 23593 +}; + +const int16_t* const WebRtcIlbcfix_kGain[3] = +{WebRtcIlbcfix_kGainSq5, WebRtcIlbcfix_kGainSq4, WebRtcIlbcfix_kGainSq3}; + + +/* Tables for the Enhancer, using upsamling factor 4 (ENH_UPS0 = 4) */ + +const int16_t WebRtcIlbcfix_kEnhPolyPhaser[ENH_UPS0][ENH_FLO_MULT2_PLUS1]={ + {0, 0, 0, 4096, 0, 0, 0}, + {64, -315, 1181, 3531, -436, 77, -64}, + {97, -509, 2464, 2464, -509, 97, -97}, + {77, -436, 3531, 1181, -315, 64, -77} +}; + +const int16_t WebRtcIlbcfix_kEnhWt[3] = { + 4800, 16384, 27968 /* Q16 */ +}; + +const size_t WebRtcIlbcfix_kEnhPlocs[ENH_NBLOCKS_TOT] = { + 160, 480, 800, 1120, 1440, 1760, 2080, 2400 /* Q(-2) */ +}; + +/* PLC table */ + +const int16_t WebRtcIlbcfix_kPlcPerSqr[6] = { /* Grid points for square of periodiciy in Q15 */ + 839, 1343, 2048, 2998, 4247, 5849 +}; + +const int16_t WebRtcIlbcfix_kPlcPitchFact[6] = { /* Value of y=(x^4-0.4)/(0.7-0.4) in grid points in Q15 */ + 0, 5462, 10922, 16384, 21846, 27306 +}; + +const int16_t WebRtcIlbcfix_kPlcPfSlope[6] = { /* Slope of y=(x^4-0.4)/(0.7-0.4) in Q11 */ + 26667, 18729, 13653, 10258, 7901, 6214 +}; diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.h new file mode 100644 index 0000000000..a8645c00db --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + constants.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CONSTANTS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CONSTANTS_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/* high pass filters */ + +extern const int16_t WebRtcIlbcfix_kHpInCoefs[]; +extern const int16_t WebRtcIlbcfix_kHpOutCoefs[]; + +/* Window for start state decision */ +extern const int16_t WebRtcIlbcfix_kStartSequenceEnrgWin[]; + +/* low pass filter used for downsampling */ +extern const int16_t WebRtcIlbcfix_kLpFiltCoefs[]; + +/* LPC analysis and quantization */ + +extern const int16_t WebRtcIlbcfix_kLpcWin[]; +extern const int16_t WebRtcIlbcfix_kLpcAsymWin[]; +extern const int32_t WebRtcIlbcfix_kLpcLagWin[]; +extern const int16_t WebRtcIlbcfix_kLpcChirpSyntDenum[]; +extern const int16_t WebRtcIlbcfix_kLpcChirpWeightDenum[]; +extern const int16_t WebRtcIlbcfix_kLsfDimCb[]; +extern const int16_t WebRtcIlbcfix_kLsfSizeCb[]; +extern const int16_t WebRtcIlbcfix_kLsfCb[]; +extern const int16_t WebRtcIlbcfix_kLsfWeight20ms[]; +extern const int16_t WebRtcIlbcfix_kLsfWeight30ms[]; +extern const int16_t WebRtcIlbcfix_kLsfMean[]; +extern const int16_t WebRtcIlbcfix_kLspMean[]; +extern const int16_t WebRtcIlbcfix_kCos[]; +extern const int16_t WebRtcIlbcfix_kCosDerivative[]; +extern const int16_t WebRtcIlbcfix_kCosGrid[]; +extern const int16_t WebRtcIlbcfix_kAcosDerivative[]; + +/* state quantization tables */ + +extern const int16_t WebRtcIlbcfix_kStateSq3[]; +extern const int32_t WebRtcIlbcfix_kChooseFrgQuant[]; +extern const int16_t WebRtcIlbcfix_kScale[]; +extern const int16_t WebRtcIlbcfix_kFrgQuantMod[]; + +/* Ranges for search and filters at different subframes */ + +extern const size_t WebRtcIlbcfix_kSearchRange[5][CB_NSTAGES]; +extern const size_t WebRtcIlbcfix_kFilterRange[]; + +/* gain quantization tables */ + +extern const int16_t WebRtcIlbcfix_kGainSq3[]; +extern const int16_t WebRtcIlbcfix_kGainSq4[]; +extern const int16_t WebRtcIlbcfix_kGainSq5[]; +extern const int16_t WebRtcIlbcfix_kGainSq5Sq[]; +extern const int16_t* const WebRtcIlbcfix_kGain[]; + +/* adaptive codebook definitions */ + +extern const int16_t WebRtcIlbcfix_kCbFiltersRev[]; +extern const int16_t WebRtcIlbcfix_kAlpha[]; + +/* enhancer definitions */ + +extern const int16_t WebRtcIlbcfix_kEnhPolyPhaser[ENH_UPS0] + [ENH_FLO_MULT2_PLUS1]; +extern const int16_t WebRtcIlbcfix_kEnhWt[]; +extern const size_t WebRtcIlbcfix_kEnhPlocs[]; + +/* PLC tables */ + +extern const int16_t WebRtcIlbcfix_kPlcPerSqr[]; +extern const int16_t WebRtcIlbcfix_kPlcPitchFact[]; +extern const int16_t WebRtcIlbcfix_kPlcPfSlope[]; + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c new file mode 100644 index 0000000000..7e21faee6c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CreateAugmentedVec.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/create_augmented_vec.h" + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "rtc_base/sanitizer.h" + +/*----------------------------------------------------------------* + * Recreate a specific codebook vector from the augmented part. + * + *----------------------------------------------------------------*/ + +void WebRtcIlbcfix_CreateAugmentedVec( + size_t index, /* (i) Index for the augmented vector to be + created */ + const int16_t* buffer, /* (i) Pointer to the end of the codebook memory + that is used for creation of the augmented + codebook */ + int16_t* cbVec) { /* (o) The constructed codebook vector */ + size_t ilow; + const int16_t *ppo, *ppi; + int16_t cbVecTmp[4]; + /* Interpolation starts 4 elements before cbVec+index, but must not start + outside `cbVec`; clamping interp_len to stay within `cbVec`. + */ + size_t interp_len = WEBRTC_SPL_MIN(index, 4); + + rtc_MsanCheckInitialized(buffer - index - interp_len, sizeof(buffer[0]), + index + interp_len); + + ilow = index - interp_len; + + /* copy the first noninterpolated part */ + ppo = buffer-index; + WEBRTC_SPL_MEMCPY_W16(cbVec, ppo, index); + + /* interpolation */ + ppo = buffer - interp_len; + ppi = buffer - index - interp_len; + + /* perform cbVec[ilow+k] = ((ppi[k]*alphaTbl[k])>>15) + + ((ppo[k]*alphaTbl[interp_len-1-k])>>15); + for k = 0..interp_len-1 + */ + WebRtcSpl_ElementwiseVectorMult(&cbVec[ilow], ppi, WebRtcIlbcfix_kAlpha, + interp_len, 15); + WebRtcSpl_ReverseOrderMultArrayElements( + cbVecTmp, ppo, &WebRtcIlbcfix_kAlpha[interp_len - 1], interp_len, 15); + WebRtcSpl_AddVectorsAndShift(&cbVec[ilow], &cbVec[ilow], cbVecTmp, interp_len, + 0); + + /* copy the second noninterpolated part */ + ppo = buffer - index; + /* `tempbuff2` is declared in WebRtcIlbcfix_GetCbVec and is SUBL+5 elements + long. `buffer` points one element past the end of that vector, i.e., at + tempbuff2+SUBL+5. Since ppo=buffer-index, we cannot read any more than + `index` elements from `ppo`. + + `cbVec` is declared to be SUBL elements long in WebRtcIlbcfix_CbConstruct. + Therefore, we can only write SUBL-index elements to cbVec+index. + + These two conditions limit the number of elements to copy. + */ + WEBRTC_SPL_MEMCPY_W16(cbVec+index, ppo, WEBRTC_SPL_MIN(SUBL-index, index)); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.h new file mode 100644 index 0000000000..d7e5be1c2f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_CreateAugmentedVec.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CREATE_AUGMENTED_VEC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_CREATE_AUGMENTED_VEC_H_ + +#include +#include + +/*----------------------------------------------------------------* + * Recreate a specific codebook vector from the augmented part. + * + *----------------------------------------------------------------*/ + +void WebRtcIlbcfix_CreateAugmentedVec( + size_t index, /* (i) Index for the augmented vector to be + created */ + const int16_t* buffer, /* (i) Pointer to the end of the codebook memory + that is used for creation of the augmented + codebook */ + int16_t* cbVec); /* (o) The construced codebook vector */ + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.c new file mode 100644 index 0000000000..d7621d5b65 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.c @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Decode.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/decode.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/decode_residual.h" +#include "modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/do_plc.h" +#include "modules/audio_coding/codecs/ilbc/enhancer_interface.h" +#include "modules/audio_coding/codecs/ilbc/hp_output.h" +#include "modules/audio_coding/codecs/ilbc/index_conv_dec.h" +#include "modules/audio_coding/codecs/ilbc/init_decode.h" +#include "modules/audio_coding/codecs/ilbc/lsf_check.h" +#include "modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h" +#include "modules/audio_coding/codecs/ilbc/unpack_bits.h" +#include "modules/audio_coding/codecs/ilbc/xcorr_coef.h" +#include "rtc_base/system/arch.h" + +#ifndef WEBRTC_ARCH_BIG_ENDIAN +#include "modules/audio_coding/codecs/ilbc/swap_bytes.h" +#endif + +/*----------------------------------------------------------------* + * main decoder function + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_DecodeImpl( + int16_t *decblock, /* (o) decoded signal block */ + const uint16_t *bytes, /* (i) encoded signal bits */ + IlbcDecoder *iLBCdec_inst, /* (i/o) the decoder state + structure */ + int16_t mode /* (i) 0: bad packet, PLC, + 1: normal */ + ) { + const int old_mode = iLBCdec_inst->mode; + const int old_use_enhancer = iLBCdec_inst->use_enhancer; + + size_t i; + int16_t order_plus_one; + + int16_t last_bit; + int16_t *data; + /* Stack based */ + int16_t decresidual[BLOCKL_MAX]; + int16_t PLCresidual[BLOCKL_MAX + LPC_FILTERORDER]; + int16_t syntdenum[NSUB_MAX*(LPC_FILTERORDER+1)]; + int16_t PLClpc[LPC_FILTERORDER + 1]; +#ifndef WEBRTC_ARCH_BIG_ENDIAN + uint16_t swapped[NO_OF_WORDS_30MS]; +#endif + iLBC_bits *iLBCbits_inst = (iLBC_bits*)PLCresidual; + + /* Reuse some buffers that are non overlapping in order to save stack memory */ + data = &PLCresidual[LPC_FILTERORDER]; + + if (mode) { /* the data are good */ + + /* decode data */ + + /* Unpacketize bits into parameters */ + +#ifndef WEBRTC_ARCH_BIG_ENDIAN + WebRtcIlbcfix_SwapBytes(bytes, iLBCdec_inst->no_of_words, swapped); + last_bit = WebRtcIlbcfix_UnpackBits(swapped, iLBCbits_inst, iLBCdec_inst->mode); +#else + last_bit = WebRtcIlbcfix_UnpackBits(bytes, iLBCbits_inst, iLBCdec_inst->mode); +#endif + + /* Check for bit errors */ + if (iLBCbits_inst->startIdx<1) + mode = 0; + if ((iLBCdec_inst->mode==20) && (iLBCbits_inst->startIdx>3)) + mode = 0; + if ((iLBCdec_inst->mode==30) && (iLBCbits_inst->startIdx>5)) + mode = 0; + if (last_bit==1) + mode = 0; + + if (mode) { /* No bit errors was detected, continue decoding */ + /* Stack based */ + int16_t lsfdeq[LPC_FILTERORDER*LPC_N_MAX]; + int16_t weightdenum[(LPC_FILTERORDER + 1)*NSUB_MAX]; + + /* adjust index */ + WebRtcIlbcfix_IndexConvDec(iLBCbits_inst->cb_index); + + /* decode the lsf */ + WebRtcIlbcfix_SimpleLsfDeQ(lsfdeq, (int16_t*)(iLBCbits_inst->lsf), iLBCdec_inst->lpc_n); + WebRtcIlbcfix_LsfCheck(lsfdeq, LPC_FILTERORDER, iLBCdec_inst->lpc_n); + WebRtcIlbcfix_DecoderInterpolateLsp(syntdenum, weightdenum, + lsfdeq, LPC_FILTERORDER, iLBCdec_inst); + + /* Decode the residual using the cb and gain indexes */ + if (!WebRtcIlbcfix_DecodeResidual(iLBCdec_inst, iLBCbits_inst, + decresidual, syntdenum)) + goto error; + + /* preparing the plc for a future loss! */ + WebRtcIlbcfix_DoThePlc( + PLCresidual, PLClpc, 0, decresidual, + syntdenum + (LPC_FILTERORDER + 1) * (iLBCdec_inst->nsub - 1), + iLBCdec_inst->last_lag, iLBCdec_inst); + + /* Use the output from doThePLC */ + WEBRTC_SPL_MEMCPY_W16(decresidual, PLCresidual, iLBCdec_inst->blockl); + } + + } + + if (mode == 0) { + /* the data is bad (either a PLC call + * was made or a bit error was detected) + */ + + /* packet loss conceal */ + + WebRtcIlbcfix_DoThePlc(PLCresidual, PLClpc, 1, decresidual, syntdenum, + iLBCdec_inst->last_lag, iLBCdec_inst); + + WEBRTC_SPL_MEMCPY_W16(decresidual, PLCresidual, iLBCdec_inst->blockl); + + order_plus_one = LPC_FILTERORDER + 1; + + for (i = 0; i < iLBCdec_inst->nsub; i++) { + WEBRTC_SPL_MEMCPY_W16(syntdenum+(i*order_plus_one), + PLClpc, order_plus_one); + } + } + + if ((*iLBCdec_inst).use_enhancer == 1) { /* Enhancer activated */ + + /* Update the filter and filter coefficients if there was a packet loss */ + if (iLBCdec_inst->prev_enh_pl==2) { + for (i=0;insub;i++) { + WEBRTC_SPL_MEMCPY_W16(&(iLBCdec_inst->old_syntdenum[i*(LPC_FILTERORDER+1)]), + syntdenum, (LPC_FILTERORDER+1)); + } + } + + /* post filtering */ + (*iLBCdec_inst).last_lag = + WebRtcIlbcfix_EnhancerInterface(data, decresidual, iLBCdec_inst); + + /* synthesis filtering */ + + /* Set up the filter state */ + WEBRTC_SPL_MEMCPY_W16(&data[-LPC_FILTERORDER], iLBCdec_inst->syntMem, LPC_FILTERORDER); + + if (iLBCdec_inst->mode==20) { + /* Enhancer has 40 samples delay */ + i=0; + WebRtcSpl_FilterARFastQ12( + data, data, + iLBCdec_inst->old_syntdenum + (i+iLBCdec_inst->nsub-1)*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + + for (i=1; i < iLBCdec_inst->nsub; i++) { + WebRtcSpl_FilterARFastQ12( + data+i*SUBL, data+i*SUBL, + syntdenum+(i-1)*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + } + + } else if (iLBCdec_inst->mode==30) { + /* Enhancer has 80 samples delay */ + for (i=0; i < 2; i++) { + WebRtcSpl_FilterARFastQ12( + data+i*SUBL, data+i*SUBL, + iLBCdec_inst->old_syntdenum + (i+4)*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + } + for (i=2; i < iLBCdec_inst->nsub; i++) { + WebRtcSpl_FilterARFastQ12( + data+i*SUBL, data+i*SUBL, + syntdenum+(i-2)*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + } + } + + /* Save the filter state */ + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->syntMem, &data[iLBCdec_inst->blockl-LPC_FILTERORDER], LPC_FILTERORDER); + + } else { /* Enhancer not activated */ + size_t lag; + + /* Find last lag (since the enhancer is not called to give this info) */ + lag = 20; + if (iLBCdec_inst->mode==20) { + lag = WebRtcIlbcfix_XcorrCoef( + &decresidual[iLBCdec_inst->blockl-60], + &decresidual[iLBCdec_inst->blockl-60-lag], + 60, + 80, lag, -1); + } else { + lag = WebRtcIlbcfix_XcorrCoef( + &decresidual[iLBCdec_inst->blockl-ENH_BLOCKL], + &decresidual[iLBCdec_inst->blockl-ENH_BLOCKL-lag], + ENH_BLOCKL, + 100, lag, -1); + } + + /* Store lag (it is needed if next packet is lost) */ + (*iLBCdec_inst).last_lag = lag; + + /* copy data and run synthesis filter */ + WEBRTC_SPL_MEMCPY_W16(data, decresidual, iLBCdec_inst->blockl); + + /* Set up the filter state */ + WEBRTC_SPL_MEMCPY_W16(&data[-LPC_FILTERORDER], iLBCdec_inst->syntMem, LPC_FILTERORDER); + + for (i=0; i < iLBCdec_inst->nsub; i++) { + WebRtcSpl_FilterARFastQ12( + data+i*SUBL, data+i*SUBL, + syntdenum + i*(LPC_FILTERORDER+1), + LPC_FILTERORDER+1, SUBL); + } + + /* Save the filter state */ + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->syntMem, &data[iLBCdec_inst->blockl-LPC_FILTERORDER], LPC_FILTERORDER); + } + + WEBRTC_SPL_MEMCPY_W16(decblock,data,iLBCdec_inst->blockl); + + /* High pass filter the signal (with upscaling a factor 2 and saturation) */ + WebRtcIlbcfix_HpOutput(decblock, (int16_t*)WebRtcIlbcfix_kHpOutCoefs, + iLBCdec_inst->hpimemy, iLBCdec_inst->hpimemx, + iLBCdec_inst->blockl); + + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->old_syntdenum, + syntdenum, iLBCdec_inst->nsub*(LPC_FILTERORDER+1)); + + iLBCdec_inst->prev_enh_pl=0; + + if (mode==0) { /* PLC was used */ + iLBCdec_inst->prev_enh_pl=1; + } + + return 0; // Success. + +error: + // The decoder got sick from eating that data. Reset it and return. + WebRtcIlbcfix_InitDecode(iLBCdec_inst, old_mode, old_use_enhancer); + return -1; // Error +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.h new file mode 100644 index 0000000000..a7d2910115 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Decode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODE_H_ + +#include + +#include "absl/base/attributes.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * main decoder function + *---------------------------------------------------------------*/ + +// Returns 0 on success, -1 on error. +ABSL_MUST_USE_RESULT +int WebRtcIlbcfix_DecodeImpl( + int16_t* decblock, /* (o) decoded signal block */ + const uint16_t* bytes, /* (i) encoded signal bits */ + IlbcDecoder* iLBCdec_inst, /* (i/o) the decoder state + structure */ + int16_t mode /* (i) 0: bad packet, PLC, + 1: normal */ +); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.c new file mode 100644 index 0000000000..a9668e2889 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.c @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DecodeResidual.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/decode_residual.h" + +#include + +#include "modules/audio_coding/codecs/ilbc/cb_construct.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/do_plc.h" +#include "modules/audio_coding/codecs/ilbc/enhancer_interface.h" +#include "modules/audio_coding/codecs/ilbc/index_conv_dec.h" +#include "modules/audio_coding/codecs/ilbc/lsf_check.h" +#include "modules/audio_coding/codecs/ilbc/state_construct.h" +#include "modules/audio_coding/codecs/ilbc/xcorr_coef.h" + +/*----------------------------------------------------------------* + * frame residual decoder function (subrutine to iLBC_decode) + *---------------------------------------------------------------*/ + +bool WebRtcIlbcfix_DecodeResidual( + IlbcDecoder *iLBCdec_inst, + /* (i/o) the decoder state structure */ + iLBC_bits *iLBC_encbits, /* (i/o) Encoded bits, which are used + for the decoding */ + int16_t *decresidual, /* (o) decoded residual frame */ + int16_t *syntdenum /* (i) the decoded synthesis filter + coefficients */ + ) { + size_t meml_gotten, diff, start_pos; + size_t subcount, subframe; + int16_t *reverseDecresidual = iLBCdec_inst->enh_buf; /* Reversed decoded data, used for decoding backwards in time (reuse memory in state) */ + int16_t *memVec = iLBCdec_inst->prevResidual; /* Memory for codebook and filter state (reuse memory in state) */ + int16_t *mem = &memVec[CB_HALFFILTERLEN]; /* Memory for codebook */ + + diff = STATE_LEN - iLBCdec_inst->state_short_len; + + if (iLBC_encbits->state_first == 1) { + start_pos = (iLBC_encbits->startIdx-1)*SUBL; + } else { + start_pos = (iLBC_encbits->startIdx-1)*SUBL + diff; + } + + /* decode scalar part of start state */ + + WebRtcIlbcfix_StateConstruct(iLBC_encbits->idxForMax, + iLBC_encbits->idxVec, &syntdenum[(iLBC_encbits->startIdx-1)*(LPC_FILTERORDER+1)], + &decresidual[start_pos], iLBCdec_inst->state_short_len + ); + + if (iLBC_encbits->state_first) { /* put adaptive part in the end */ + + /* setup memory */ + + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - iLBCdec_inst->state_short_len); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-iLBCdec_inst->state_short_len, decresidual+start_pos, + iLBCdec_inst->state_short_len); + + /* construct decoded vector */ + + if (!WebRtcIlbcfix_CbConstruct( + &decresidual[start_pos + iLBCdec_inst->state_short_len], + iLBC_encbits->cb_index, iLBC_encbits->gain_index, + mem + CB_MEML - ST_MEM_L_TBL, ST_MEM_L_TBL, diff)) + return false; // Error. + + } + else {/* put adaptive part in the beginning */ + + /* setup memory */ + + meml_gotten = iLBCdec_inst->state_short_len; + WebRtcSpl_MemCpyReversedOrder(mem+CB_MEML-1, + decresidual+start_pos, meml_gotten); + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - meml_gotten); + + /* construct decoded vector */ + + if (!WebRtcIlbcfix_CbConstruct(reverseDecresidual, iLBC_encbits->cb_index, + iLBC_encbits->gain_index, + mem + CB_MEML - ST_MEM_L_TBL, ST_MEM_L_TBL, + diff)) + return false; // Error. + + /* get decoded residual from reversed vector */ + + WebRtcSpl_MemCpyReversedOrder(&decresidual[start_pos-1], + reverseDecresidual, diff); + } + + /* counter for predicted subframes */ + + subcount=1; + + /* forward prediction of subframes */ + + if (iLBCdec_inst->nsub > iLBC_encbits->startIdx + 1) { + + /* setup memory */ + WebRtcSpl_MemSetW16(mem, 0, CB_MEML-STATE_LEN); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-STATE_LEN, + decresidual+(iLBC_encbits->startIdx-1)*SUBL, STATE_LEN); + + /* loop over subframes to encode */ + + size_t Nfor = iLBCdec_inst->nsub - iLBC_encbits->startIdx - 1; + for (subframe=0; subframestartIdx + 1 + subframe) * SUBL], + iLBC_encbits->cb_index + subcount * CB_NSTAGES, + iLBC_encbits->gain_index + subcount * CB_NSTAGES, mem, MEM_LF_TBL, + SUBL)) + return false; // Error; + + /* update memory */ + memmove(mem, mem + SUBL, (CB_MEML - SUBL) * sizeof(*mem)); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-SUBL, + &decresidual[(iLBC_encbits->startIdx+1+subframe)*SUBL], SUBL); + + subcount++; + } + + } + + /* backward prediction of subframes */ + + if (iLBC_encbits->startIdx > 1) { + + /* setup memory */ + + meml_gotten = SUBL*(iLBCdec_inst->nsub+1-iLBC_encbits->startIdx); + if( meml_gotten > CB_MEML ) { + meml_gotten=CB_MEML; + } + + WebRtcSpl_MemCpyReversedOrder(mem+CB_MEML-1, + decresidual+(iLBC_encbits->startIdx-1)*SUBL, meml_gotten); + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - meml_gotten); + + /* loop over subframes to decode */ + + size_t Nback = iLBC_encbits->startIdx - 1; + for (subframe=0; subframecb_index + subcount * CB_NSTAGES, + iLBC_encbits->gain_index + subcount * CB_NSTAGES, mem, MEM_LF_TBL, + SUBL)) + return false; // Error. + + /* update memory */ + memmove(mem, mem + SUBL, (CB_MEML - SUBL) * sizeof(*mem)); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-SUBL, + &reverseDecresidual[subframe*SUBL], SUBL); + + subcount++; + } + + /* get decoded residual from reversed vector */ + WebRtcSpl_MemCpyReversedOrder(decresidual+SUBL*Nback-1, + reverseDecresidual, SUBL*Nback); + } + + return true; // Success. +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.h new file mode 100644 index 0000000000..d079577661 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DecodeResidual.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODE_RESIDUAL_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODE_RESIDUAL_H_ + +#include +#include +#include + +#include "absl/base/attributes.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * frame residual decoder function (subrutine to iLBC_decode) + *---------------------------------------------------------------*/ + +// Returns true on success, false on failure. In case of failure, the decoder +// state may be corrupted and needs resetting. +ABSL_MUST_USE_RESULT +bool WebRtcIlbcfix_DecodeResidual( + IlbcDecoder* iLBCdec_inst, /* (i/o) the decoder state structure */ + iLBC_bits* iLBC_encbits, /* (i/o) Encoded bits, which are used + for the decoding */ + int16_t* decresidual, /* (o) decoded residual frame */ + int16_t* syntdenum /* (i) the decoded synthesis filter + coefficients */ +); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c new file mode 100644 index 0000000000..d96bb9b2e9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DecoderInterpolateLsp.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h" + +#include "modules/audio_coding/codecs/ilbc/bw_expand.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h" + +/*----------------------------------------------------------------* + * obtain synthesis and weighting filters form lsf coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_DecoderInterpolateLsp( + int16_t *syntdenum, /* (o) synthesis filter coefficients */ + int16_t *weightdenum, /* (o) weighting denumerator + coefficients */ + int16_t *lsfdeq, /* (i) dequantized lsf coefficients */ + int16_t length, /* (i) length of lsf coefficient vector */ + IlbcDecoder *iLBCdec_inst + /* (i) the decoder state structure */ + ){ + size_t i; + int pos, lp_length; + int16_t lp[LPC_FILTERORDER + 1], *lsfdeq2; + + lsfdeq2 = lsfdeq + length; + lp_length = length + 1; + + if (iLBCdec_inst->mode==30) { + /* subframe 1: Interpolation between old and first LSF */ + + WebRtcIlbcfix_LspInterpolate2PolyDec(lp, (*iLBCdec_inst).lsfdeqold, lsfdeq, + WebRtcIlbcfix_kLsfWeight30ms[0], length); + WEBRTC_SPL_MEMCPY_W16(syntdenum,lp,lp_length); + WebRtcIlbcfix_BwExpand(weightdenum, lp, (int16_t*)WebRtcIlbcfix_kLpcChirpSyntDenum, (int16_t)lp_length); + + /* subframes 2 to 6: interpolation between first and last LSF */ + + pos = lp_length; + for (i = 1; i < 6; i++) { + WebRtcIlbcfix_LspInterpolate2PolyDec(lp, lsfdeq, lsfdeq2, + WebRtcIlbcfix_kLsfWeight30ms[i], length); + WEBRTC_SPL_MEMCPY_W16(syntdenum + pos,lp,lp_length); + WebRtcIlbcfix_BwExpand(weightdenum + pos, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpSyntDenum, (int16_t)lp_length); + pos += lp_length; + } + } else { /* iLBCdec_inst->mode=20 */ + /* subframes 1 to 4: interpolation between old and new LSF */ + pos = 0; + for (i = 0; i < iLBCdec_inst->nsub; i++) { + WebRtcIlbcfix_LspInterpolate2PolyDec(lp, iLBCdec_inst->lsfdeqold, lsfdeq, + WebRtcIlbcfix_kLsfWeight20ms[i], length); + WEBRTC_SPL_MEMCPY_W16(syntdenum+pos,lp,lp_length); + WebRtcIlbcfix_BwExpand(weightdenum+pos, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpSyntDenum, (int16_t)lp_length); + pos += lp_length; + } + } + + /* update memory */ + + if (iLBCdec_inst->mode==30) { + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->lsfdeqold, lsfdeq2, length); + } else { + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->lsfdeqold, lsfdeq, length); + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h new file mode 100644 index 0000000000..8b08114467 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DecoderInterpolateLsp.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODER_INTERPOLATE_LSF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DECODER_INTERPOLATE_LSF_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * obtain synthesis and weighting filters form lsf coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_DecoderInterpolateLsp( + int16_t* syntdenum, /* (o) synthesis filter coefficients */ + int16_t* weightdenum, /* (o) weighting denumerator + coefficients */ + int16_t* lsfdeq, /* (i) dequantized lsf coefficients */ + int16_t length, /* (i) length of lsf coefficient vector */ + IlbcDecoder* iLBCdec_inst + /* (i) the decoder state structure */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/defines.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/defines.h new file mode 100644 index 0000000000..64135c4887 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/defines.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + define.h + +******************************************************************/ +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DEFINES_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DEFINES_H_ + +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" + +/* general codec settings */ + +#define FS 8000 +#define BLOCKL_20MS 160 +#define BLOCKL_30MS 240 +#define BLOCKL_MAX 240 +#define NSUB_20MS 4 +#define NSUB_30MS 6 +#define NSUB_MAX 6 +#define NASUB_20MS 2 +#define NASUB_30MS 4 +#define NASUB_MAX 4 +#define SUBL 40 +#define STATE_LEN 80 +#define STATE_SHORT_LEN_30MS 58 +#define STATE_SHORT_LEN_20MS 57 + +/* LPC settings */ + +#define LPC_FILTERORDER 10 +#define LPC_LOOKBACK 60 +#define LPC_N_20MS 1 +#define LPC_N_30MS 2 +#define LPC_N_MAX 2 +#define LPC_ASYMDIFF 20 +#define LSF_NSPLIT 3 +#define LSF_NUMBER_OF_STEPS 4 +#define LPC_HALFORDER 5 +#define COS_GRID_POINTS 60 + +/* cb settings */ + +#define CB_NSTAGES 3 +#define CB_EXPAND 2 +#define CB_MEML 147 +#define CB_FILTERLEN (2 * 4) +#define CB_HALFFILTERLEN 4 +#define CB_RESRANGE 34 +#define CB_MAXGAIN_FIXQ6 83 /* error = -0.24% */ +#define CB_MAXGAIN_FIXQ14 21299 + +/* enhancer */ + +#define ENH_BLOCKL 80 /* block length */ +#define ENH_BLOCKL_HALF (ENH_BLOCKL / 2) +#define ENH_HL \ + 3 /* 2*ENH_HL+1 is number blocks \ + in said second \ + sequence */ +#define ENH_SLOP \ + 2 /* max difference estimated and \ + correct pitch period */ +#define ENH_PLOCSL \ + 8 /* pitch-estimates and \ + pitch-locations buffer \ + length */ +#define ENH_OVERHANG 2 +#define ENH_UPS0 4 /* upsampling rate */ +#define ENH_FL0 3 /* 2*FLO+1 is the length of each filter */ +#define ENH_FLO_MULT2_PLUS1 7 +#define ENH_VECTL (ENH_BLOCKL + 2 * ENH_FL0) +#define ENH_CORRDIM (2 * ENH_SLOP + 1) +#define ENH_NBLOCKS (BLOCKL / ENH_BLOCKL) +#define ENH_NBLOCKS_EXTRA 5 +#define ENH_NBLOCKS_TOT 8 /* ENH_NBLOCKS+ENH_NBLOCKS_EXTRA */ +#define ENH_BUFL (ENH_NBLOCKS_TOT) * ENH_BLOCKL +#define ENH_BUFL_FILTEROVERHEAD 3 +#define ENH_A0 819 /* Q14 */ +#define ENH_A0_MINUS_A0A0DIV4 848256041 /* Q34 */ +#define ENH_A0DIV2 26843546 /* Q30 */ + +/* PLC */ + +/* Down sampling */ + +#define FILTERORDER_DS_PLUS1 7 +#define DELAY_DS 3 +#define FACTOR_DS 2 + +/* bit stream defs */ + +#define NO_OF_BYTES_20MS 38 +#define NO_OF_BYTES_30MS 50 +#define NO_OF_WORDS_20MS 19 +#define NO_OF_WORDS_30MS 25 +#define STATE_BITS 3 +#define BYTE_LEN 8 +#define ULP_CLASSES 3 + +/* help parameters */ + +#define TWO_PI_FIX 25736 /* Q12 */ + +/* Constants for codebook search and creation */ + +#define ST_MEM_L_TBL 85 +#define MEM_LF_TBL 147 + +/* Struct for the bits */ +typedef struct iLBC_bits_t_ { + int16_t lsf[LSF_NSPLIT * LPC_N_MAX]; + int16_t cb_index[CB_NSTAGES * (NASUB_MAX + 1)]; /* First CB_NSTAGES values + contains extra CB index */ + int16_t gain_index[CB_NSTAGES * (NASUB_MAX + 1)]; /* First CB_NSTAGES values + contains extra CB gain */ + size_t idxForMax; + int16_t state_first; + int16_t idxVec[STATE_SHORT_LEN_30MS]; + int16_t firstbits; + size_t startIdx; +} iLBC_bits; + +/* type definition encoder instance */ +typedef struct IlbcEncoder_ { + /* flag for frame size mode */ + int16_t mode; + + /* basic parameters for different frame sizes */ + size_t blockl; + size_t nsub; + int16_t nasub; + size_t no_of_bytes, no_of_words; + int16_t lpc_n; + size_t state_short_len; + + /* analysis filter state */ + int16_t anaMem[LPC_FILTERORDER]; + + /* Fix-point old lsf parameters for interpolation */ + int16_t lsfold[LPC_FILTERORDER]; + int16_t lsfdeqold[LPC_FILTERORDER]; + + /* signal buffer for LP analysis */ + int16_t lpc_buffer[LPC_LOOKBACK + BLOCKL_MAX]; + + /* state of input HP filter */ + int16_t hpimemx[2]; + int16_t hpimemy[4]; + +#ifdef SPLIT_10MS + int16_t weightdenumbuf[66]; + int16_t past_samples[160]; + uint16_t bytes[25]; + int16_t section; + int16_t Nfor_flag; + int16_t Nback_flag; + int16_t start_pos; + size_t diff; +#endif + +} IlbcEncoder; + +/* type definition decoder instance */ +typedef struct IlbcDecoder_ { + /* flag for frame size mode */ + int16_t mode; + + /* basic parameters for different frame sizes */ + size_t blockl; + size_t nsub; + int16_t nasub; + size_t no_of_bytes, no_of_words; + int16_t lpc_n; + size_t state_short_len; + + /* synthesis filter state */ + int16_t syntMem[LPC_FILTERORDER]; + + /* old LSF for interpolation */ + int16_t lsfdeqold[LPC_FILTERORDER]; + + /* pitch lag estimated in enhancer and used in PLC */ + size_t last_lag; + + /* PLC state information */ + int consPLICount, prev_enh_pl; + int16_t perSquare; + + int16_t prevScale, prevPLI; + size_t prevLag; + int16_t prevLpc[LPC_FILTERORDER + 1]; + int16_t prevResidual[NSUB_MAX * SUBL]; + int16_t seed; + + /* previous synthesis filter parameters */ + + int16_t old_syntdenum[(LPC_FILTERORDER + 1) * NSUB_MAX]; + + /* state of output HP filter */ + int16_t hpimemx[2]; + int16_t hpimemy[4]; + + /* enhancer state information */ + int use_enhancer; + int16_t enh_buf[ENH_BUFL + ENH_BUFL_FILTEROVERHEAD]; + size_t enh_period[ENH_NBLOCKS_TOT]; + +} IlbcDecoder; + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.c new file mode 100644 index 0000000000..9ca6ca48e9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.c @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DoThePlc.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/do_plc.h" + +#include "modules/audio_coding/codecs/ilbc/bw_expand.h" +#include "modules/audio_coding/codecs/ilbc/comp_corr.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Packet loss concealment routine. Conceals a residual signal + * and LP parameters. If no packet loss, update state. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_DoThePlc( + int16_t *PLCresidual, /* (o) concealed residual */ + int16_t *PLClpc, /* (o) concealed LP parameters */ + int16_t PLI, /* (i) packet loss indicator + 0 - no PL, 1 = PL */ + int16_t *decresidual, /* (i) decoded residual */ + int16_t *lpc, /* (i) decoded LPC (only used for no PL) */ + size_t inlag, /* (i) pitch lag */ + IlbcDecoder *iLBCdec_inst + /* (i/o) decoder instance */ + ){ + size_t i; + int32_t cross, ener, cross_comp, ener_comp = 0; + int32_t measure, maxMeasure, energy; + int32_t noise_energy_threshold_30dB; + int16_t max, crossSquareMax, crossSquare; + size_t j, lag, randlag; + int16_t tmp1, tmp2; + int16_t shift1, shift2, shift3, shiftMax; + int16_t scale3; + size_t corrLen; + int32_t tmpW32, tmp2W32; + int16_t use_gain; + int16_t tot_gain; + int16_t max_perSquare; + int16_t scale1, scale2; + int16_t totscale; + int32_t nom; + int16_t denom; + int16_t pitchfact; + size_t use_lag; + int ind; + int16_t randvec[BLOCKL_MAX]; + + /* Packet Loss */ + if (PLI == 1) { + + (*iLBCdec_inst).consPLICount += 1; + + /* if previous frame not lost, + determine pitch pred. gain */ + + if (iLBCdec_inst->prevPLI != 1) { + + /* Maximum 60 samples are correlated, preserve as high accuracy + as possible without getting overflow */ + max = WebRtcSpl_MaxAbsValueW16((*iLBCdec_inst).prevResidual, + iLBCdec_inst->blockl); + scale3 = (WebRtcSpl_GetSizeInBits(max)<<1) - 25; + if (scale3 < 0) { + scale3 = 0; + } + + /* Store scale for use when interpolating between the + * concealment and the received packet */ + iLBCdec_inst->prevScale = scale3; + + /* Search around the previous lag +/-3 to find the + best pitch period */ + lag = inlag - 3; + + /* Guard against getting outside the frame */ + corrLen = (size_t)WEBRTC_SPL_MIN(60, iLBCdec_inst->blockl-(inlag+3)); + + WebRtcIlbcfix_CompCorr( &cross, &ener, + iLBCdec_inst->prevResidual, lag, iLBCdec_inst->blockl, corrLen, scale3); + + /* Normalize and store cross^2 and the number of shifts */ + shiftMax = WebRtcSpl_GetSizeInBits(WEBRTC_SPL_ABS_W32(cross))-15; + crossSquareMax = (int16_t)(( + (int16_t)WEBRTC_SPL_SHIFT_W32(cross, -shiftMax) * + (int16_t)WEBRTC_SPL_SHIFT_W32(cross, -shiftMax)) >> 15); + + for (j=inlag-2;j<=inlag+3;j++) { + WebRtcIlbcfix_CompCorr( &cross_comp, &ener_comp, + iLBCdec_inst->prevResidual, j, iLBCdec_inst->blockl, corrLen, scale3); + + /* Use the criteria (corr*corr)/energy to compare if + this lag is better or not. To avoid the division, + do a cross multiplication */ + shift1 = WebRtcSpl_GetSizeInBits(WEBRTC_SPL_ABS_W32(cross_comp))-15; + crossSquare = (int16_t)(( + (int16_t)WEBRTC_SPL_SHIFT_W32(cross_comp, -shift1) * + (int16_t)WEBRTC_SPL_SHIFT_W32(cross_comp, -shift1)) >> 15); + + shift2 = WebRtcSpl_GetSizeInBits(ener)-15; + measure = (int16_t)WEBRTC_SPL_SHIFT_W32(ener, -shift2) * crossSquare; + + shift3 = WebRtcSpl_GetSizeInBits(ener_comp)-15; + maxMeasure = (int16_t)WEBRTC_SPL_SHIFT_W32(ener_comp, -shift3) * + crossSquareMax; + + /* Calculate shift value, so that the two measures can + be put in the same Q domain */ + if(2 * shiftMax + shift3 > 2 * shift1 + shift2) { + tmp1 = + WEBRTC_SPL_MIN(31, 2 * shiftMax + shift3 - 2 * shift1 - shift2); + tmp2 = 0; + } else { + tmp1 = 0; + tmp2 = + WEBRTC_SPL_MIN(31, 2 * shift1 + shift2 - 2 * shiftMax - shift3); + } + + if ((measure>>tmp1) > (maxMeasure>>tmp2)) { + /* New lag is better => record lag, measure and domain */ + lag = j; + crossSquareMax = crossSquare; + cross = cross_comp; + shiftMax = shift1; + ener = ener_comp; + } + } + + /* Calculate the periodicity for the lag with the maximum correlation. + + Definition of the periodicity: + abs(corr(vec1, vec2))/(sqrt(energy(vec1))*sqrt(energy(vec2))) + + Work in the Square domain to simplify the calculations + max_perSquare is less than 1 (in Q15) + */ + tmp2W32=WebRtcSpl_DotProductWithScale(&iLBCdec_inst->prevResidual[iLBCdec_inst->blockl-corrLen], + &iLBCdec_inst->prevResidual[iLBCdec_inst->blockl-corrLen], + corrLen, scale3); + + if ((tmp2W32>0)&&(ener_comp>0)) { + /* norm energies to int16_t, compute the product of the energies and + use the upper int16_t as the denominator */ + + scale1=(int16_t)WebRtcSpl_NormW32(tmp2W32)-16; + tmp1=(int16_t)WEBRTC_SPL_SHIFT_W32(tmp2W32, scale1); + + scale2=(int16_t)WebRtcSpl_NormW32(ener)-16; + tmp2=(int16_t)WEBRTC_SPL_SHIFT_W32(ener, scale2); + denom = (int16_t)((tmp1 * tmp2) >> 16); /* in Q(scale1+scale2-16) */ + + /* Square the cross correlation and norm it such that max_perSquare + will be in Q15 after the division */ + + totscale = scale1+scale2-1; + tmp1 = (int16_t)WEBRTC_SPL_SHIFT_W32(cross, (totscale>>1)); + tmp2 = (int16_t)WEBRTC_SPL_SHIFT_W32(cross, totscale-(totscale>>1)); + + nom = tmp1 * tmp2; + max_perSquare = (int16_t)WebRtcSpl_DivW32W16(nom, denom); + + } else { + max_perSquare = 0; + } + } + + /* previous frame lost, use recorded lag and gain */ + + else { + lag = iLBCdec_inst->prevLag; + max_perSquare = iLBCdec_inst->perSquare; + } + + /* Attenuate signal and scale down pitch pred gain if + several frames lost consecutively */ + + use_gain = 32767; /* 1.0 in Q15 */ + + if (iLBCdec_inst->consPLICount*iLBCdec_inst->blockl>320) { + use_gain = 29491; /* 0.9 in Q15 */ + } else if (iLBCdec_inst->consPLICount*iLBCdec_inst->blockl>640) { + use_gain = 22938; /* 0.7 in Q15 */ + } else if (iLBCdec_inst->consPLICount*iLBCdec_inst->blockl>960) { + use_gain = 16384; /* 0.5 in Q15 */ + } else if (iLBCdec_inst->consPLICount*iLBCdec_inst->blockl>1280) { + use_gain = 0; /* 0.0 in Q15 */ + } + + /* Compute mixing factor of picth repeatition and noise: + for max_per>0.7 set periodicity to 1.0 + 0.47868) { /* periodicity > 0.7 (0.7^4=0.2401 in Q15) */ + pitchfact = 32767; + } else if (max_perSquare>839) { /* 0.4 < periodicity < 0.7 (0.4^4=0.0256 in Q15) */ + /* find best index and interpolate from that */ + ind = 5; + while ((max_perSquare0)) { + ind--; + } + /* pitch fact is approximated by first order */ + tmpW32 = (int32_t)WebRtcIlbcfix_kPlcPitchFact[ind] + + ((WebRtcIlbcfix_kPlcPfSlope[ind] * + (max_perSquare - WebRtcIlbcfix_kPlcPerSqr[ind])) >> 11); + + pitchfact = (int16_t)WEBRTC_SPL_MIN(tmpW32, 32767); /* guard against overflow */ + + } else { /* periodicity < 0.4 */ + pitchfact = 0; + } + + /* avoid repetition of same pitch cycle (buzzyness) */ + use_lag = lag; + if (lag<80) { + use_lag = 2*lag; + } + + /* compute concealed residual */ + noise_energy_threshold_30dB = (int32_t)iLBCdec_inst->blockl * 900; + energy = 0; + for (i=0; iblockl; i++) { + + /* noise component - 52 < randlagFIX < 117 */ + iLBCdec_inst->seed = (int16_t)(iLBCdec_inst->seed * 31821 + 13849); + randlag = 53 + (iLBCdec_inst->seed & 63); + if (randlag > i) { + randvec[i] = + iLBCdec_inst->prevResidual[iLBCdec_inst->blockl + i - randlag]; + } else { + randvec[i] = iLBCdec_inst->prevResidual[i - randlag]; + } + + /* pitch repeatition component */ + if (use_lag > i) { + PLCresidual[i] = + iLBCdec_inst->prevResidual[iLBCdec_inst->blockl + i - use_lag]; + } else { + PLCresidual[i] = PLCresidual[i - use_lag]; + } + + /* Attinuate total gain for each 10 ms */ + if (i<80) { + tot_gain=use_gain; + } else if (i<160) { + tot_gain = (int16_t)((31130 * use_gain) >> 15); /* 0.95*use_gain */ + } else { + tot_gain = (int16_t)((29491 * use_gain) >> 15); /* 0.9*use_gain */ + } + + + /* mix noise and pitch repeatition */ + PLCresidual[i] = (int16_t)((tot_gain * + ((pitchfact * PLCresidual[i] + (32767 - pitchfact) * randvec[i] + + 16384) >> 15)) >> 15); + + /* Compute energy until threshold for noise energy is reached */ + if (energy < noise_energy_threshold_30dB) { + energy += PLCresidual[i] * PLCresidual[i]; + } + } + + /* less than 30 dB, use only noise */ + if (energy < noise_energy_threshold_30dB) { + for (i=0; iblockl; i++) { + PLCresidual[i] = randvec[i]; + } + } + + /* use the old LPC */ + WEBRTC_SPL_MEMCPY_W16(PLClpc, (*iLBCdec_inst).prevLpc, LPC_FILTERORDER+1); + + /* Update state in case there are multiple frame losses */ + iLBCdec_inst->prevLag = lag; + iLBCdec_inst->perSquare = max_perSquare; + } + + /* no packet loss, copy input */ + + else { + WEBRTC_SPL_MEMCPY_W16(PLCresidual, decresidual, iLBCdec_inst->blockl); + WEBRTC_SPL_MEMCPY_W16(PLClpc, lpc, (LPC_FILTERORDER+1)); + iLBCdec_inst->consPLICount = 0; + } + + /* update state */ + iLBCdec_inst->prevPLI = PLI; + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->prevLpc, PLClpc, (LPC_FILTERORDER+1)); + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->prevResidual, PLCresidual, iLBCdec_inst->blockl); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.h new file mode 100644 index 0000000000..c19c4eca32 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_DoThePlc.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DO_PLC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_DO_PLC_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Packet loss concealment routine. Conceals a residual signal + * and LP parameters. If no packet loss, update state. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_DoThePlc( + int16_t* PLCresidual, /* (o) concealed residual */ + int16_t* PLClpc, /* (o) concealed LP parameters */ + int16_t PLI, /* (i) packet loss indicator + 0 - no PL, 1 = PL */ + int16_t* decresidual, /* (i) decoded residual */ + int16_t* lpc, /* (i) decoded LPC (only used for no PL) */ + size_t inlag, /* (i) pitch lag */ + IlbcDecoder* iLBCdec_inst + /* (i/o) decoder instance */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.c new file mode 100644 index 0000000000..8e536221cd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.c @@ -0,0 +1,517 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Encode.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/encode.h" + +#include + +#include "modules/audio_coding/codecs/ilbc/cb_construct.h" +#include "modules/audio_coding/codecs/ilbc/cb_search.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/frame_classify.h" +#include "modules/audio_coding/codecs/ilbc/hp_input.h" +#include "modules/audio_coding/codecs/ilbc/index_conv_enc.h" +#include "modules/audio_coding/codecs/ilbc/lpc_encode.h" +#include "modules/audio_coding/codecs/ilbc/pack_bits.h" +#include "modules/audio_coding/codecs/ilbc/state_construct.h" +#include "modules/audio_coding/codecs/ilbc/state_search.h" +#include "rtc_base/checks.h" +#include "rtc_base/system/arch.h" + +#ifdef SPLIT_10MS +#include "modules/audio_coding/codecs/ilbc/unpack_bits.h" +#include "modules/audio_coding/codecs/ilbc/index_conv_dec.h" +#endif + +#ifndef WEBRTC_ARCH_BIG_ENDIAN +#include "modules/audio_coding/codecs/ilbc/swap_bytes.h" +#endif + +/*----------------------------------------------------------------* + * main encoder function + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_EncodeImpl( + uint16_t *bytes, /* (o) encoded data bits iLBC */ + const int16_t *block, /* (i) speech vector to encode */ + IlbcEncoder *iLBCenc_inst /* (i/o) the general encoder + state */ + ){ + size_t n, meml_gotten, Nfor; + size_t diff, start_pos; + size_t index; + size_t subcount, subframe; + size_t start_count, end_count; + int16_t *residual; + int32_t en1, en2; + int16_t scale, max; + int16_t *syntdenum; + int16_t *decresidual; + int16_t *reverseResidual; + int16_t *reverseDecresidual; + /* Stack based */ + int16_t weightdenum[(LPC_FILTERORDER + 1)*NSUB_MAX]; + int16_t dataVec[BLOCKL_MAX + LPC_FILTERORDER]; + int16_t memVec[CB_MEML+CB_FILTERLEN]; + int16_t bitsMemory[sizeof(iLBC_bits)/sizeof(int16_t)]; + iLBC_bits *iLBCbits_inst = (iLBC_bits*)bitsMemory; + + +#ifdef SPLIT_10MS + int16_t *weightdenumbuf = iLBCenc_inst->weightdenumbuf; + int16_t last_bit; +#endif + + int16_t *data = &dataVec[LPC_FILTERORDER]; + int16_t *mem = &memVec[CB_HALFFILTERLEN]; + + /* Reuse som buffers to save stack memory */ + residual = &iLBCenc_inst->lpc_buffer[LPC_LOOKBACK+BLOCKL_MAX-iLBCenc_inst->blockl]; + syntdenum = mem; /* syntdenum[(LPC_FILTERORDER + 1)*NSUB_MAX] and mem are used non overlapping in the code */ + decresidual = residual; /* Already encoded residual is overwritten by the decoded version */ + reverseResidual = data; /* data and reverseResidual are used non overlapping in the code */ + reverseDecresidual = reverseResidual; /* Already encoded residual is overwritten by the decoded version */ + +#ifdef SPLIT_10MS + + WebRtcSpl_MemSetW16 ( (int16_t *) iLBCbits_inst, 0, + sizeof(iLBC_bits) / sizeof(int16_t) ); + + start_pos = iLBCenc_inst->start_pos; + diff = iLBCenc_inst->diff; + + if (iLBCenc_inst->section != 0){ + WEBRTC_SPL_MEMCPY_W16 (weightdenum, weightdenumbuf, + SCRATCH_ENCODE_DATAVEC - SCRATCH_ENCODE_WEIGHTDENUM); + /* Un-Packetize the frame into parameters */ + last_bit = WebRtcIlbcfix_UnpackBits (iLBCenc_inst->bytes, iLBCbits_inst, iLBCenc_inst->mode); + if (last_bit) + return; + /* adjust index */ + WebRtcIlbcfix_IndexConvDec (iLBCbits_inst->cb_index); + + if (iLBCenc_inst->section == 1){ + /* Save first 80 samples of a 160/240 sample frame for 20/30msec */ + WEBRTC_SPL_MEMCPY_W16 (iLBCenc_inst->past_samples, block, 80); + } + else{ // iLBCenc_inst->section == 2 AND mode = 30ms + /* Save second 80 samples of a 240 sample frame for 30msec */ + WEBRTC_SPL_MEMCPY_W16 (iLBCenc_inst->past_samples + 80, block, 80); + } + } + else{ // iLBCenc_inst->section == 0 + /* form a complete frame of 160/240 for 20msec/30msec mode */ + WEBRTC_SPL_MEMCPY_W16 (data + (iLBCenc_inst->mode * 8) - 80, block, 80); + WEBRTC_SPL_MEMCPY_W16 (data, iLBCenc_inst->past_samples, + (iLBCenc_inst->mode * 8) - 80); + iLBCenc_inst->Nfor_flag = 0; + iLBCenc_inst->Nback_flag = 0; +#else + /* copy input block to data*/ + WEBRTC_SPL_MEMCPY_W16(data,block,iLBCenc_inst->blockl); +#endif + + /* high pass filtering of input signal and scale down the residual (*0.5) */ + WebRtcIlbcfix_HpInput(data, (int16_t*)WebRtcIlbcfix_kHpInCoefs, + iLBCenc_inst->hpimemy, iLBCenc_inst->hpimemx, + iLBCenc_inst->blockl); + + /* LPC of hp filtered input data */ + WebRtcIlbcfix_LpcEncode(syntdenum, weightdenum, iLBCbits_inst->lsf, data, + iLBCenc_inst); + + /* Set up state */ + WEBRTC_SPL_MEMCPY_W16(dataVec, iLBCenc_inst->anaMem, LPC_FILTERORDER); + + /* inverse filter to get residual */ + for (n=0; nnsub; n++ ) { + WebRtcSpl_FilterMAFastQ12( + &data[n*SUBL], &residual[n*SUBL], + &syntdenum[n*(LPC_FILTERORDER+1)], + LPC_FILTERORDER+1, SUBL); + } + + /* Copy the state for next frame */ + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->anaMem, &data[iLBCenc_inst->blockl-LPC_FILTERORDER], LPC_FILTERORDER); + + /* find state location */ + + iLBCbits_inst->startIdx = WebRtcIlbcfix_FrameClassify(iLBCenc_inst,residual); + + /* check if state should be in first or last part of the + two subframes */ + + index = (iLBCbits_inst->startIdx-1)*SUBL; + max=WebRtcSpl_MaxAbsValueW16(&residual[index], 2*SUBL); + scale = WebRtcSpl_GetSizeInBits((uint32_t)(max * max)); + + /* Scale to maximum 25 bits so that the MAC won't cause overflow */ + scale = scale - 25; + if(scale < 0) { + scale = 0; + } + + diff = STATE_LEN - iLBCenc_inst->state_short_len; + en1=WebRtcSpl_DotProductWithScale(&residual[index], &residual[index], + iLBCenc_inst->state_short_len, scale); + index += diff; + en2=WebRtcSpl_DotProductWithScale(&residual[index], &residual[index], + iLBCenc_inst->state_short_len, scale); + if (en1 > en2) { + iLBCbits_inst->state_first = 1; + start_pos = (iLBCbits_inst->startIdx-1)*SUBL; + } else { + iLBCbits_inst->state_first = 0; + start_pos = (iLBCbits_inst->startIdx-1)*SUBL + diff; + } + + /* scalar quantization of state */ + + WebRtcIlbcfix_StateSearch(iLBCenc_inst, iLBCbits_inst, &residual[start_pos], + &syntdenum[(iLBCbits_inst->startIdx-1)*(LPC_FILTERORDER+1)], + &weightdenum[(iLBCbits_inst->startIdx-1)*(LPC_FILTERORDER+1)]); + + WebRtcIlbcfix_StateConstruct(iLBCbits_inst->idxForMax, iLBCbits_inst->idxVec, + &syntdenum[(iLBCbits_inst->startIdx-1)*(LPC_FILTERORDER+1)], + &decresidual[start_pos], iLBCenc_inst->state_short_len + ); + + /* predictive quantization in state */ + + if (iLBCbits_inst->state_first) { /* put adaptive part in the end */ + + /* setup memory */ + + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - iLBCenc_inst->state_short_len); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-iLBCenc_inst->state_short_len, + decresidual+start_pos, iLBCenc_inst->state_short_len); + + /* encode subframes */ + + WebRtcIlbcfix_CbSearch(iLBCenc_inst, iLBCbits_inst->cb_index, iLBCbits_inst->gain_index, + &residual[start_pos+iLBCenc_inst->state_short_len], + mem+CB_MEML-ST_MEM_L_TBL, ST_MEM_L_TBL, diff, + &weightdenum[iLBCbits_inst->startIdx*(LPC_FILTERORDER+1)], 0); + + /* construct decoded vector */ + + RTC_CHECK(WebRtcIlbcfix_CbConstruct( + &decresidual[start_pos + iLBCenc_inst->state_short_len], + iLBCbits_inst->cb_index, iLBCbits_inst->gain_index, + mem + CB_MEML - ST_MEM_L_TBL, ST_MEM_L_TBL, diff)); + + } + else { /* put adaptive part in the beginning */ + + /* create reversed vectors for prediction */ + + WebRtcSpl_MemCpyReversedOrder(&reverseResidual[diff-1], + &residual[(iLBCbits_inst->startIdx+1)*SUBL-STATE_LEN], diff); + + /* setup memory */ + + meml_gotten = iLBCenc_inst->state_short_len; + WebRtcSpl_MemCpyReversedOrder(&mem[CB_MEML-1], &decresidual[start_pos], meml_gotten); + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - iLBCenc_inst->state_short_len); + + /* encode subframes */ + WebRtcIlbcfix_CbSearch(iLBCenc_inst, iLBCbits_inst->cb_index, iLBCbits_inst->gain_index, + reverseResidual, mem+CB_MEML-ST_MEM_L_TBL, ST_MEM_L_TBL, diff, + &weightdenum[(iLBCbits_inst->startIdx-1)*(LPC_FILTERORDER+1)], + 0); + + /* construct decoded vector */ + RTC_CHECK(WebRtcIlbcfix_CbConstruct( + reverseDecresidual, iLBCbits_inst->cb_index, + iLBCbits_inst->gain_index, mem + CB_MEML - ST_MEM_L_TBL, + ST_MEM_L_TBL, diff)); + + /* get decoded residual from reversed vector */ + + WebRtcSpl_MemCpyReversedOrder(&decresidual[start_pos-1], reverseDecresidual, diff); + } + +#ifdef SPLIT_10MS + iLBCenc_inst->start_pos = start_pos; + iLBCenc_inst->diff = diff; + iLBCenc_inst->section++; + /* adjust index */ + WebRtcIlbcfix_IndexConvEnc (iLBCbits_inst->cb_index); + /* Packetize the parameters into the frame */ + WebRtcIlbcfix_PackBits (iLBCenc_inst->bytes, iLBCbits_inst, iLBCenc_inst->mode); + WEBRTC_SPL_MEMCPY_W16 (weightdenumbuf, weightdenum, + SCRATCH_ENCODE_DATAVEC - SCRATCH_ENCODE_WEIGHTDENUM); + return; + } +#endif + + /* forward prediction of subframes */ + + Nfor = iLBCenc_inst->nsub-iLBCbits_inst->startIdx-1; + + /* counter for predicted subframes */ +#ifdef SPLIT_10MS + if (iLBCenc_inst->mode == 20) + { + subcount = 1; + } + if (iLBCenc_inst->mode == 30) + { + if (iLBCenc_inst->section == 1) + { + subcount = 1; + } + if (iLBCenc_inst->section == 2) + { + subcount = 3; + } + } +#else + subcount=1; +#endif + + if( Nfor > 0 ){ + + /* setup memory */ + + WebRtcSpl_MemSetW16(mem, 0, CB_MEML-STATE_LEN); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-STATE_LEN, + decresidual+(iLBCbits_inst->startIdx-1)*SUBL, STATE_LEN); + +#ifdef SPLIT_10MS + if (iLBCenc_inst->Nfor_flag > 0) + { + for (subframe = 0; subframe < WEBRTC_SPL_MIN (Nfor, 2); subframe++) + { + /* update memory */ + WEBRTC_SPL_MEMCPY_W16 (mem, mem + SUBL, (CB_MEML - SUBL)); + WEBRTC_SPL_MEMCPY_W16 (mem + CB_MEML - SUBL, + &decresidual[(iLBCbits_inst->startIdx + 1 + + subframe) * SUBL], SUBL); + } + } + + iLBCenc_inst->Nfor_flag++; + + if (iLBCenc_inst->mode == 20) + { + start_count = 0; + end_count = Nfor; + } + if (iLBCenc_inst->mode == 30) + { + if (iLBCenc_inst->section == 1) + { + start_count = 0; + end_count = WEBRTC_SPL_MIN (Nfor, (size_t)2); + } + if (iLBCenc_inst->section == 2) + { + start_count = WEBRTC_SPL_MIN (Nfor, (size_t)2); + end_count = Nfor; + } + } +#else + start_count = 0; + end_count = Nfor; +#endif + + /* loop over subframes to encode */ + + for (subframe = start_count; subframe < end_count; subframe++){ + + /* encode subframe */ + + WebRtcIlbcfix_CbSearch(iLBCenc_inst, iLBCbits_inst->cb_index+subcount*CB_NSTAGES, + iLBCbits_inst->gain_index+subcount*CB_NSTAGES, + &residual[(iLBCbits_inst->startIdx+1+subframe)*SUBL], + mem, MEM_LF_TBL, SUBL, + &weightdenum[(iLBCbits_inst->startIdx+1+subframe)*(LPC_FILTERORDER+1)], + subcount); + + /* construct decoded vector */ + RTC_CHECK(WebRtcIlbcfix_CbConstruct( + &decresidual[(iLBCbits_inst->startIdx + 1 + subframe) * SUBL], + iLBCbits_inst->cb_index + subcount * CB_NSTAGES, + iLBCbits_inst->gain_index + subcount * CB_NSTAGES, mem, MEM_LF_TBL, + SUBL)); + + /* update memory */ + + memmove(mem, mem + SUBL, (CB_MEML - SUBL) * sizeof(*mem)); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-SUBL, + &decresidual[(iLBCbits_inst->startIdx+1+subframe)*SUBL], SUBL); + + subcount++; + } + } + +#ifdef SPLIT_10MS + if ((iLBCenc_inst->section == 1) && + (iLBCenc_inst->mode == 30) && (Nfor > 0) && (end_count == 2)) + { + iLBCenc_inst->section++; + /* adjust index */ + WebRtcIlbcfix_IndexConvEnc (iLBCbits_inst->cb_index); + /* Packetize the parameters into the frame */ + WebRtcIlbcfix_PackBits (iLBCenc_inst->bytes, iLBCbits_inst, iLBCenc_inst->mode); + WEBRTC_SPL_MEMCPY_W16 (weightdenumbuf, weightdenum, + SCRATCH_ENCODE_DATAVEC - SCRATCH_ENCODE_WEIGHTDENUM); + return; + } +#endif + + /* backward prediction of subframes */ + + if (iLBCbits_inst->startIdx > 1) { + + /* create reverse order vectors + (The decresidual does not need to be copied since it is + contained in the same vector as the residual) + */ + + size_t Nback = iLBCbits_inst->startIdx - 1; + WebRtcSpl_MemCpyReversedOrder(&reverseResidual[Nback*SUBL-1], residual, Nback*SUBL); + + /* setup memory */ + + meml_gotten = SUBL*(iLBCenc_inst->nsub+1-iLBCbits_inst->startIdx); + if( meml_gotten > CB_MEML ) { + meml_gotten=CB_MEML; + } + + WebRtcSpl_MemCpyReversedOrder(&mem[CB_MEML-1], &decresidual[Nback*SUBL], meml_gotten); + WebRtcSpl_MemSetW16(mem, 0, CB_MEML - meml_gotten); + +#ifdef SPLIT_10MS + if (iLBCenc_inst->Nback_flag > 0) + { + for (subframe = 0; subframe < WEBRTC_SPL_MAX (2 - Nfor, 0); subframe++) + { + /* update memory */ + WEBRTC_SPL_MEMCPY_W16 (mem, mem + SUBL, (CB_MEML - SUBL)); + WEBRTC_SPL_MEMCPY_W16 (mem + CB_MEML - SUBL, + &reverseDecresidual[subframe * SUBL], SUBL); + } + } + + iLBCenc_inst->Nback_flag++; + + + if (iLBCenc_inst->mode == 20) + { + start_count = 0; + end_count = Nback; + } + if (iLBCenc_inst->mode == 30) + { + if (iLBCenc_inst->section == 1) + { + start_count = 0; + end_count = (Nfor >= 2) ? 0 : (2 - NFor); + } + if (iLBCenc_inst->section == 2) + { + start_count = (Nfor >= 2) ? 0 : (2 - NFor); + end_count = Nback; + } + } +#else + start_count = 0; + end_count = Nback; +#endif + + /* loop over subframes to encode */ + + for (subframe = start_count; subframe < end_count; subframe++){ + + /* encode subframe */ + + WebRtcIlbcfix_CbSearch(iLBCenc_inst, iLBCbits_inst->cb_index+subcount*CB_NSTAGES, + iLBCbits_inst->gain_index+subcount*CB_NSTAGES, &reverseResidual[subframe*SUBL], + mem, MEM_LF_TBL, SUBL, + &weightdenum[(iLBCbits_inst->startIdx-2-subframe)*(LPC_FILTERORDER+1)], + subcount); + + /* construct decoded vector */ + RTC_CHECK(WebRtcIlbcfix_CbConstruct( + &reverseDecresidual[subframe * SUBL], + iLBCbits_inst->cb_index + subcount * CB_NSTAGES, + iLBCbits_inst->gain_index + subcount * CB_NSTAGES, mem, MEM_LF_TBL, + SUBL)); + + /* update memory */ + memmove(mem, mem + SUBL, (CB_MEML - SUBL) * sizeof(*mem)); + WEBRTC_SPL_MEMCPY_W16(mem+CB_MEML-SUBL, + &reverseDecresidual[subframe*SUBL], SUBL); + + subcount++; + + } + + /* get decoded residual from reversed vector */ + + WebRtcSpl_MemCpyReversedOrder(&decresidual[SUBL*Nback-1], reverseDecresidual, SUBL*Nback); + } + /* end encoding part */ + + /* adjust index */ + + WebRtcIlbcfix_IndexConvEnc(iLBCbits_inst->cb_index); + + /* Packetize the parameters into the frame */ + +#ifdef SPLIT_10MS + if( (iLBCenc_inst->mode==30) && (iLBCenc_inst->section==1) ){ + WebRtcIlbcfix_PackBits(iLBCenc_inst->bytes, iLBCbits_inst, iLBCenc_inst->mode); + } + else{ + WebRtcIlbcfix_PackBits(bytes, iLBCbits_inst, iLBCenc_inst->mode); + } +#else + WebRtcIlbcfix_PackBits(bytes, iLBCbits_inst, iLBCenc_inst->mode); +#endif + +#ifndef WEBRTC_ARCH_BIG_ENDIAN + /* Swap bytes for LITTLE ENDIAN since the packbits() + function assumes BIG_ENDIAN machine */ +#ifdef SPLIT_10MS + if (( (iLBCenc_inst->section == 1) && (iLBCenc_inst->mode == 20) ) || + ( (iLBCenc_inst->section == 2) && (iLBCenc_inst->mode == 30) )){ + WebRtcIlbcfix_SwapBytes(bytes, iLBCenc_inst->no_of_words, bytes); + } +#else + WebRtcIlbcfix_SwapBytes(bytes, iLBCenc_inst->no_of_words, bytes); +#endif +#endif + +#ifdef SPLIT_10MS + if (subcount == (iLBCenc_inst->nsub - 1)) + { + iLBCenc_inst->section = 0; + } + else + { + iLBCenc_inst->section++; + WEBRTC_SPL_MEMCPY_W16 (weightdenumbuf, weightdenum, + SCRATCH_ENCODE_DATAVEC - SCRATCH_ENCODE_WEIGHTDENUM); + } +#endif + +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.h new file mode 100644 index 0000000000..bc3e187d92 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Encode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ENCODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_ENCODE_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * main encoder function + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_EncodeImpl( + uint16_t* bytes, /* (o) encoded data bits iLBC */ + const int16_t* block, /* (i) speech vector to encode */ + IlbcEncoder* iLBCenc_inst /* (i/o) the general encoder + state */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c new file mode 100644 index 0000000000..7f00254aea --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_EnergyInverse.c + +******************************************************************/ + +/* Inverses the in vector in into Q29 domain */ + +#include "modules/audio_coding/codecs/ilbc/energy_inverse.h" + +void WebRtcIlbcfix_EnergyInverse( + int16_t *energy, /* (i/o) Energy and inverse + energy (in Q29) */ + size_t noOfEnergies) /* (i) The length of the energy + vector */ +{ + int32_t Nom=(int32_t)0x1FFFFFFF; + int16_t *energyPtr; + size_t i; + + /* Set the minimum energy value to 16384 to avoid overflow */ + energyPtr=energy; + for (i=0; i +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/* Inverses the in vector in into Q29 domain */ + +void WebRtcIlbcfix_EnergyInverse( + int16_t* + energy, /* (i/o) Energy and inverse + energy (in Q29) */ + size_t noOfEnergies); /* (i) The length of the energy + vector */ + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enh_upsample.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enh_upsample.c new file mode 100644 index 0000000000..cd3d0a4db1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enh_upsample.c @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_EnhUpsample.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/enh_upsample.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * upsample finite array assuming zeros outside bounds + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_EnhUpsample( + int32_t *useq1, /* (o) upsampled output sequence */ + int16_t *seq1 /* (i) unupsampled sequence */ + ){ + int j; + int32_t *pu1, *pu11; + int16_t *ps, *w16tmp; + const int16_t *pp; + + /* filtering: filter overhangs left side of sequence */ + pu1=useq1; + for (j=0;j +#include + +/*----------------------------------------------------------------* + * perform enhancement on idata+centerStartPos through + * idata+centerStartPos+ENH_BLOCKL-1 + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Enhancer( + int16_t* odata, /* (o) smoothed block, dimension blockl */ + int16_t* idata, /* (i) data buffer used for enhancing */ + size_t idatal, /* (i) dimension idata */ + size_t centerStartPos, /* (i) first sample current block within idata */ + size_t* period, /* (i) pitch period array (pitch bward-in time) */ + const size_t* plocs, /* (i) locations where period array values valid */ + size_t periodl /* (i) dimension of period and plocs */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.c new file mode 100644 index 0000000000..ca23e19ae3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.c @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_EnhancerInterface.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/enhancer_interface.h" + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/enhancer.h" +#include "modules/audio_coding/codecs/ilbc/hp_output.h" +#include "modules/audio_coding/codecs/ilbc/xcorr_coef.h" + + + +/*----------------------------------------------------------------* + * interface for enhancer + *---------------------------------------------------------------*/ + +size_t // (o) Estimated lag in end of in[] + WebRtcIlbcfix_EnhancerInterface( + int16_t* out, // (o) enhanced signal + const int16_t* in, // (i) unenhanced signal + IlbcDecoder* iLBCdec_inst) { // (i) buffers etc + size_t iblock; + size_t lag=20, tlag=20; + size_t inLen=iLBCdec_inst->blockl+120; + int16_t scale, scale1; + size_t plc_blockl; + int16_t *enh_buf; + size_t *enh_period; + int32_t tmp1, tmp2, max; + size_t new_blocks; + int16_t *enh_bufPtr1; + size_t i; + size_t k; + int16_t EnChange; + int16_t SqrtEnChange; + int16_t inc; + int16_t win; + int16_t *tmpW16ptr; + size_t startPos; + int16_t *plc_pred; + const int16_t *target, *regressor; + int16_t max16; + int shifts; + int32_t ener; + int16_t enerSh; + int16_t corrSh; + size_t ind; + int16_t sh; + size_t start, stop; + /* Stack based */ + int16_t totsh[3]; + int16_t downsampled[(BLOCKL_MAX+120)>>1]; /* length 180 */ + int32_t corr32[50]; + int32_t corrmax[3]; + int16_t corr16[3]; + int16_t en16[3]; + size_t lagmax[3]; + + plc_pred = downsampled; /* Reuse memory since plc_pred[ENH_BLOCKL] and + downsampled are non overlapping */ + enh_buf=iLBCdec_inst->enh_buf; + enh_period=iLBCdec_inst->enh_period; + + /* Copy in the new data into the enhancer buffer */ + memmove(enh_buf, &enh_buf[iLBCdec_inst->blockl], + (ENH_BUFL - iLBCdec_inst->blockl) * sizeof(*enh_buf)); + + WEBRTC_SPL_MEMCPY_W16(&enh_buf[ENH_BUFL-iLBCdec_inst->blockl], in, + iLBCdec_inst->blockl); + + /* Set variables that are dependent on frame size */ + if (iLBCdec_inst->mode==30) { + plc_blockl=ENH_BLOCKL; + new_blocks=3; + startPos=320; /* Start position for enhancement + (640-new_blocks*ENH_BLOCKL-80) */ + } else { + plc_blockl=40; + new_blocks=2; + startPos=440; /* Start position for enhancement + (640-new_blocks*ENH_BLOCKL-40) */ + } + + /* Update the pitch prediction for each enhancer block, move the old ones */ + memmove(enh_period, &enh_period[new_blocks], + (ENH_NBLOCKS_TOT - new_blocks) * sizeof(*enh_period)); + + WebRtcSpl_DownsampleFast( + enh_buf+ENH_BUFL-inLen, /* Input samples */ + inLen + ENH_BUFL_FILTEROVERHEAD, + downsampled, + inLen / 2, + (int16_t*)WebRtcIlbcfix_kLpFiltCoefs, /* Coefficients in Q12 */ + FILTERORDER_DS_PLUS1, /* Length of filter (order-1) */ + FACTOR_DS, + DELAY_DS); + + /* Estimate the pitch in the down sampled domain. */ + for(iblock = 0; iblock> 16); + en16[i] = (int16_t)WEBRTC_SPL_SHIFT_W32(ener, enerSh); + totsh[i] = enerSh - 2 * corrSh; + } + + /* Compare lagmax[0..3] for the (corr^2)/ener criteria */ + ind = 0; + for (i=1; i<3; i++) { + if (totsh[ind] > totsh[i]) { + sh = WEBRTC_SPL_MIN(31, totsh[ind]-totsh[i]); + if (corr16[ind] * en16[i] < (corr16[i] * en16[ind]) >> sh) { + ind = i; + } + } else { + sh = WEBRTC_SPL_MIN(31, totsh[i]-totsh[ind]); + if ((corr16[ind] * en16[i]) >> sh < corr16[i] * en16[ind]) { + ind = i; + } + } + } + + lag = lagmax[ind] + 10; + + /* Store the estimated lag in the non-downsampled domain */ + enh_period[ENH_NBLOCKS_TOT - new_blocks + iblock] = lag * 8; + + /* Store the estimated lag for backward PLC */ + if (iLBCdec_inst->prev_enh_pl==1) { + if (!iblock) { + tlag = lag * 2; + } + } else { + if (iblock==1) { + tlag = lag * 2; + } + } + + lag *= 2; + } + + if ((iLBCdec_inst->prev_enh_pl==1)||(iLBCdec_inst->prev_enh_pl==2)) { + + /* Calculate the best lag of the new frame + This is used to interpolate backwards and mix with the PLC'd data + */ + + /* references */ + target=in; + regressor=in+tlag-1; + + /* scaling */ + // Note that this is not abs-max, so we will take the absolute value below. + max16 = WebRtcSpl_MaxAbsElementW16(regressor, plc_blockl + 3 - 1); + const int16_t max_target = + WebRtcSpl_MaxAbsElementW16(target, plc_blockl + 3 - 1); + const int64_t max_val = plc_blockl * abs(max16 * max_target); + const int32_t factor = max_val >> 31; + shifts = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); + + /* compute cross correlation */ + WebRtcSpl_CrossCorrelation(corr32, target, regressor, plc_blockl, 3, shifts, + 1); + + /* find lag */ + lag=WebRtcSpl_MaxIndexW32(corr32, 3); + lag+=tlag-1; + + /* Copy the backward PLC to plc_pred */ + + if (iLBCdec_inst->prev_enh_pl==1) { + if (lag>plc_blockl) { + WEBRTC_SPL_MEMCPY_W16(plc_pred, &in[lag-plc_blockl], plc_blockl); + } else { + WEBRTC_SPL_MEMCPY_W16(&plc_pred[plc_blockl-lag], in, lag); + WEBRTC_SPL_MEMCPY_W16( + plc_pred, &enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl+lag], + (plc_blockl-lag)); + } + } else { + size_t pos; + + pos = plc_blockl; + + while (lagprev_enh_pl==1) { + /* limit energy change + if energy in backward PLC is more than 4 times higher than the forward + PLC, then reduce the energy in the backward PLC vector: + sample 1...len-16 set energy of the to 4 times forward PLC + sample len-15..len interpolate between 4 times fw PLC and bw PLC energy + + Note: Compared to floating point code there is a slight change, + the window is 16 samples long instead of 10 samples to simplify the + calculations + */ + + max=WebRtcSpl_MaxAbsValueW16( + &enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl], plc_blockl); + max16=WebRtcSpl_MaxAbsValueW16(plc_pred, plc_blockl); + max = WEBRTC_SPL_MAX(max, max16); + scale=22-(int16_t)WebRtcSpl_NormW32(max); + scale=WEBRTC_SPL_MAX(scale,0); + + tmp2 = WebRtcSpl_DotProductWithScale( + &enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl], + &enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl], + plc_blockl, scale); + tmp1 = WebRtcSpl_DotProductWithScale(plc_pred, plc_pred, + plc_blockl, scale); + + /* Check the energy difference */ + if ((tmp1>0)&&((tmp1>>2)>tmp2)) { + /* EnChange is now guaranteed to be <0.5 + Calculate EnChange=tmp2/tmp1 in Q16 + */ + + scale1=(int16_t)WebRtcSpl_NormW32(tmp1); + tmp1=WEBRTC_SPL_SHIFT_W32(tmp1, (scale1-16)); /* using 15 bits */ + + tmp2=WEBRTC_SPL_SHIFT_W32(tmp2, (scale1)); + EnChange = (int16_t)WebRtcSpl_DivW32W16(tmp2, + (int16_t)tmp1); + + /* Calculate the Sqrt of the energy in Q15 ((14+16)/2) */ + SqrtEnChange = (int16_t)WebRtcSpl_SqrtFloor(EnChange << 14); + + + /* Multiply first part of vector with 2*SqrtEnChange */ + WebRtcSpl_ScaleVector(plc_pred, plc_pred, SqrtEnChange, plc_blockl-16, + 14); + + /* Calculate increase parameter for window part (16 last samples) */ + /* (1-2*SqrtEnChange)/16 in Q15 */ + inc = 2048 - (SqrtEnChange >> 3); + + win=0; + tmpW16ptr=&plc_pred[plc_blockl-16]; + + for (i=16;i>0;i--) { + *tmpW16ptr = (int16_t)( + (*tmpW16ptr * (SqrtEnChange + (win >> 1))) >> 14); + /* multiply by (2.0*SqrtEnChange+win) */ + + win += inc; + tmpW16ptr++; + } + } + + /* Make the linear interpolation between the forward PLC'd data + and the backward PLC'd data (from the new frame) + */ + + if (plc_blockl==40) { + inc=400; /* 1/41 in Q14 */ + } else { /* plc_blockl==80 */ + inc=202; /* 1/81 in Q14 */ + } + win=0; + enh_bufPtr1=&enh_buf[ENH_BUFL-1-iLBCdec_inst->blockl]; + for (i=0; i> 14); + *enh_bufPtr1 += (int16_t)( + ((16384 - win) * plc_pred[plc_blockl - 1 - i]) >> 14); + enh_bufPtr1--; + } + } else { + int16_t *synt = &downsampled[LPC_FILTERORDER]; + + enh_bufPtr1=&enh_buf[ENH_BUFL-iLBCdec_inst->blockl-plc_blockl]; + WEBRTC_SPL_MEMCPY_W16(enh_bufPtr1, plc_pred, plc_blockl); + + /* Clear fileter memory */ + WebRtcSpl_MemSetW16(iLBCdec_inst->syntMem, 0, LPC_FILTERORDER); + WebRtcSpl_MemSetW16(iLBCdec_inst->hpimemy, 0, 4); + WebRtcSpl_MemSetW16(iLBCdec_inst->hpimemx, 0, 2); + + /* Initialize filter memory by filtering through 2 lags */ + WEBRTC_SPL_MEMCPY_W16(&synt[-LPC_FILTERORDER], iLBCdec_inst->syntMem, + LPC_FILTERORDER); + WebRtcSpl_FilterARFastQ12( + enh_bufPtr1, + synt, + &iLBCdec_inst->old_syntdenum[ + (iLBCdec_inst->nsub-1)*(LPC_FILTERORDER+1)], + LPC_FILTERORDER+1, lag); + + WEBRTC_SPL_MEMCPY_W16(&synt[-LPC_FILTERORDER], &synt[lag-LPC_FILTERORDER], + LPC_FILTERORDER); + WebRtcIlbcfix_HpOutput(synt, (int16_t*)WebRtcIlbcfix_kHpOutCoefs, + iLBCdec_inst->hpimemy, iLBCdec_inst->hpimemx, + lag); + WebRtcSpl_FilterARFastQ12( + enh_bufPtr1, synt, + &iLBCdec_inst->old_syntdenum[ + (iLBCdec_inst->nsub-1)*(LPC_FILTERORDER+1)], + LPC_FILTERORDER+1, lag); + + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->syntMem, &synt[lag-LPC_FILTERORDER], + LPC_FILTERORDER); + WebRtcIlbcfix_HpOutput(synt, (int16_t*)WebRtcIlbcfix_kHpOutCoefs, + iLBCdec_inst->hpimemy, iLBCdec_inst->hpimemx, + lag); + } + } + + + /* Perform enhancement block by block */ + + for (iblock = 0; iblock +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * interface for enhancer + *---------------------------------------------------------------*/ + +size_t // (o) Estimated lag in end of in[] +WebRtcIlbcfix_EnhancerInterface(int16_t* out, // (o) enhanced signal + const int16_t* in, // (i) unenhanced signal + IlbcDecoder* iLBCdec_inst); // (i) buffers etc + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.c new file mode 100644 index 0000000000..6b4f30c96b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_FilteredCbVecs.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/filtered_cb_vecs.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Construct an additional codebook vector by filtering the + * initial codebook buffer. This vector is then used to expand + * the codebook with an additional section. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_FilteredCbVecs( + int16_t *cbvectors, /* (o) Codebook vector for the higher section */ + int16_t *CBmem, /* (i) Codebook memory that is filtered to create a + second CB section */ + size_t lMem, /* (i) Length of codebook memory */ + size_t samples /* (i) Number of samples to filter */ + ) { + + /* Set up the memory, start with zero state */ + WebRtcSpl_MemSetW16(CBmem+lMem, 0, CB_HALFFILTERLEN); + WebRtcSpl_MemSetW16(CBmem-CB_HALFFILTERLEN, 0, CB_HALFFILTERLEN); + WebRtcSpl_MemSetW16(cbvectors, 0, lMem-samples); + + /* Filter to obtain the filtered CB memory */ + + WebRtcSpl_FilterMAFastQ12( + CBmem+CB_HALFFILTERLEN+lMem-samples, cbvectors+lMem-samples, + (int16_t*)WebRtcIlbcfix_kCbFiltersRev, CB_FILTERLEN, samples); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.h new file mode 100644 index 0000000000..661262e42e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_FilteredCbVecs.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_FILTERED_CB_VECS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_FILTERED_CB_VECS_H_ + +#include +#include + +/*----------------------------------------------------------------* + * Construct an additional codebook vector by filtering the + * initial codebook buffer. This vector is then used to expand + * the codebook with an additional section. + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_FilteredCbVecs( + int16_t* cbvectors, /* (o) Codebook vector for the higher section */ + int16_t* CBmem, /* (i) Codebook memory that is filtered to create a + second CB section */ + size_t lMem, /* (i) Length of codebook memory */ + size_t samples /* (i) Number of samples to filter */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.c new file mode 100644 index 0000000000..c1084b1645 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_FrameClassify.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/frame_classify.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Classification of subframes to localize start state + *---------------------------------------------------------------*/ + +size_t WebRtcIlbcfix_FrameClassify( + /* (o) Index to the max-energy sub frame */ + IlbcEncoder *iLBCenc_inst, + /* (i/o) the encoder state structure */ + int16_t *residualFIX /* (i) lpc residual signal */ + ){ + int16_t max, scale; + int32_t ssqEn[NSUB_MAX-1]; + int16_t *ssqPtr; + int32_t *seqEnPtr; + int32_t maxW32; + int16_t scale1; + size_t pos; + size_t n; + + /* + Calculate the energy of each of the 80 sample blocks + in the draft the 4 first and last samples are windowed with 1/5...4/5 + and 4/5...1/5 respectively. To simplify for the fixpoint we have changed + this to 0 0 1 1 and 1 1 0 0 + */ + + max = WebRtcSpl_MaxAbsValueW16(residualFIX, iLBCenc_inst->blockl); + scale = WebRtcSpl_GetSizeInBits((uint32_t)(max * max)); + + /* Scale to maximum 24 bits so that it won't overflow for 76 samples */ + scale = scale-24; + scale1 = WEBRTC_SPL_MAX(0, scale); + + /* Calculate energies */ + ssqPtr=residualFIX + 2; + seqEnPtr=ssqEn; + for (n=(iLBCenc_inst->nsub-1); n>0; n--) { + (*seqEnPtr) = WebRtcSpl_DotProductWithScale(ssqPtr, ssqPtr, 76, scale1); + ssqPtr += 40; + seqEnPtr++; + } + + /* Scale to maximum 20 bits in order to allow for the 11 bit window */ + maxW32 = WebRtcSpl_MaxValueW32(ssqEn, iLBCenc_inst->nsub - 1); + scale = WebRtcSpl_GetSizeInBits(maxW32) - 20; + scale1 = WEBRTC_SPL_MAX(0, scale); + + /* Window each 80 block with the ssqEn_winTbl window to give higher probability for + the blocks in the middle + */ + seqEnPtr=ssqEn; + if (iLBCenc_inst->mode==20) { + ssqPtr=(int16_t*)WebRtcIlbcfix_kStartSequenceEnrgWin+1; + } else { + ssqPtr=(int16_t*)WebRtcIlbcfix_kStartSequenceEnrgWin; + } + for (n=(iLBCenc_inst->nsub-1); n>0; n--) { + (*seqEnPtr)=WEBRTC_SPL_MUL(((*seqEnPtr)>>scale1), (*ssqPtr)); + seqEnPtr++; + ssqPtr++; + } + + /* Extract the best choise of start state */ + pos = WebRtcSpl_MaxIndexW32(ssqEn, iLBCenc_inst->nsub - 1) + 1; + + return(pos); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.h new file mode 100644 index 0000000000..7615106d70 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_FrameClassify.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_FRAME_CLASSIFY_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_FRAME_CLASSIFY_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +size_t WebRtcIlbcfix_FrameClassify( + /* (o) Index to the max-energy sub frame */ + IlbcEncoder* iLBCenc_inst, + /* (i/o) the encoder state structure */ + int16_t* residualFIX /* (i) lpc residual signal */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c new file mode 100644 index 0000000000..1357dece33 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GainDequant.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/gain_dequant.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * decoder for quantized gains in the gain-shape coding of + * residual + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_GainDequant( + /* (o) quantized gain value (Q14) */ + int16_t index, /* (i) quantization index */ + int16_t maxIn, /* (i) maximum of unquantized gain (Q14) */ + int16_t stage /* (i) The stage of the search */ + ){ + int16_t scale; + const int16_t *gain; + + /* obtain correct scale factor */ + + scale=WEBRTC_SPL_ABS_W16(maxIn); + scale = WEBRTC_SPL_MAX(1638, scale); /* if lower than 0.1, set it to 0.1 */ + + /* select the quantization table and return the decoded value */ + gain = WebRtcIlbcfix_kGain[stage]; + + return (int16_t)((scale * gain[index] + 8192) >> 14); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.h new file mode 100644 index 0000000000..2b97550b6c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GainDequant.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GAIN_DEQUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GAIN_DEQUANT_H_ + +#include + +/*----------------------------------------------------------------* + * decoder for quantized gains in the gain-shape coding of + * residual + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_GainDequant( + /* (o) quantized gain value (Q14) */ + int16_t index, /* (i) quantization index */ + int16_t maxIn, /* (i) maximum of unquantized gain (Q14) */ + int16_t stage /* (i) The stage of the search */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.c new file mode 100644 index 0000000000..9a6d49d51a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GainQuant.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/gain_quant.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * quantizer for the gain in the gain-shape coding of residual + *---------------------------------------------------------------*/ + +int16_t WebRtcIlbcfix_GainQuant( /* (o) quantized gain value */ + int16_t gain, /* (i) gain value Q14 */ + int16_t maxIn, /* (i) maximum of gain value Q14 */ + int16_t stage, /* (i) The stage of the search */ + int16_t *index /* (o) quantization index */ + ) { + + int16_t scale, cblen; + int32_t gainW32, measure1, measure2; + const int16_t *cbPtr, *cb; + int loc, noMoves, noChecks, i; + + /* ensure a lower bound (0.1) on the scaling factor */ + + scale = WEBRTC_SPL_MAX(1638, maxIn); + + /* select the quantization table and calculate + the length of the table and the number of + steps in the binary search that are needed */ + cb = WebRtcIlbcfix_kGain[stage]; + cblen = 32>>stage; + noChecks = 4-stage; + + /* Multiply the gain with 2^14 to make the comparison + easier and with higher precision */ + gainW32 = gain << 14; + + /* Do a binary search, starting in the middle of the CB + loc - defines the current position in the table + noMoves - defines the number of steps to move in the CB in order + to get next CB location + */ + + loc = cblen>>1; + noMoves = loc; + cbPtr = cb + loc; /* Centre of CB */ + + for (i=noChecks;i>0;i--) { + noMoves>>=1; + measure1 = scale * *cbPtr; + + /* Move up if gain is larger, otherwise move down in table */ + measure1 = measure1 - gainW32; + + if (0>measure1) { + cbPtr+=noMoves; + loc+=noMoves; + } else { + cbPtr-=noMoves; + loc-=noMoves; + } + } + + /* Check which value is the closest one: loc-1, loc or loc+1 */ + + measure1 = scale * *cbPtr; + if (gainW32>measure1) { + /* Check against value above loc */ + measure2 = scale * cbPtr[1]; + if ((measure2-gainW32)<(gainW32-measure1)) { + loc+=1; + } + } else { + /* Check against value below loc */ + measure2 = scale * cbPtr[-1]; + if ((gainW32-measure2)<=(measure1-gainW32)) { + loc-=1; + } + } + + /* Guard against getting outside the table. The calculation above can give a location + which is one above the maximum value (in very rare cases) */ + loc=WEBRTC_SPL_MIN(loc, (cblen-1)); + *index=loc; + + /* Calculate and return the quantized gain value (in Q14) */ + return (int16_t)((scale * cb[loc] + 8192) >> 14); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.h new file mode 100644 index 0000000000..761f7d2f79 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GainQuant.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GAIN_QUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GAIN_QUANT_H_ + +#include + +/*----------------------------------------------------------------* + * quantizer for the gain in the gain-shape coding of residual + *---------------------------------------------------------------*/ + +int16_t +WebRtcIlbcfix_GainQuant( /* (o) quantized gain value */ + int16_t gain, /* (i) gain value Q14 */ + int16_t maxIn, /* (i) maximum of gain value Q14 */ + int16_t stage, /* (i) The stage of the search */ + int16_t* index /* (o) quantization index */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c new file mode 100644 index 0000000000..e9cd2008e0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetCbVec.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/get_cd_vec.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/create_augmented_vec.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Construct codebook vector for given index. + *---------------------------------------------------------------*/ + +bool WebRtcIlbcfix_GetCbVec( + int16_t *cbvec, /* (o) Constructed codebook vector */ + int16_t *mem, /* (i) Codebook buffer */ + size_t index, /* (i) Codebook index */ + size_t lMem, /* (i) Length of codebook buffer */ + size_t cbveclen /* (i) Codebook vector length */ + ){ + size_t k, base_size; + size_t lag; + /* Stack based */ + int16_t tempbuff2[SUBL+5]; + + /* Determine size of codebook sections */ + + base_size=lMem-cbveclen+1; + + if (cbveclen==SUBL) { + base_size += cbveclen / 2; + } + + /* No filter -> First codebook section */ + + if (index +#include +#include + +#include "absl/base/attributes.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +// Returns true on success, false on failure. In case of failure, the decoder +// state may be corrupted and needs resetting. +ABSL_MUST_USE_RESULT +bool WebRtcIlbcfix_GetCbVec( + int16_t* cbvec, /* (o) Constructed codebook vector */ + int16_t* mem, /* (i) Codebook buffer */ + size_t index, /* (i) Codebook index */ + size_t lMem, /* (i) Length of codebook buffer */ + size_t cbveclen /* (i) Codebook vector length */ +); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c new file mode 100644 index 0000000000..e0fb21caf0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetLspPoly.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/get_lsp_poly.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Construct the polynomials F1(z) and F2(z) from the LSP + * (Computations are done in Q24) + * + * The expansion is performed using the following recursion: + * + * f[0] = 1; + * tmp = -2.0 * lsp[0]; + * f[1] = tmp; + * for (i=2; i<=5; i++) { + * b = -2.0 * lsp[2*i-2]; + * f[i] = tmp*f[i-1] + 2.0*f[i-2]; + * for (j=i; j>=2; j--) { + * f[j] = f[j] + tmp*f[j-1] + f[j-2]; + * } + * f[i] = f[i] + tmp; + * } + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_GetLspPoly( + int16_t *lsp, /* (i) LSP in Q15 */ + int32_t *f) /* (o) polonymial in Q24 */ +{ + int32_t tmpW32; + int i, j; + int16_t high, low; + int16_t *lspPtr; + int32_t *fPtr; + + lspPtr = lsp; + fPtr = f; + /* f[0] = 1.0 (Q24) */ + (*fPtr) = (int32_t)16777216; + fPtr++; + + (*fPtr) = WEBRTC_SPL_MUL((*lspPtr), -1024); + fPtr++; + lspPtr+=2; + + for(i=2; i<=5; i++) + { + (*fPtr) = fPtr[-2]; + + for(j=i; j>1; j--) + { + /* Compute f[j] = f[j] + tmp*f[j-1] + f[j-2]; */ + high = (int16_t)(fPtr[-1] >> 16); + low = (int16_t)((fPtr[-1] & 0xffff) >> 1); + + tmpW32 = 4 * high * *lspPtr + 4 * ((low * *lspPtr) >> 15); + + (*fPtr) += fPtr[-2]; + (*fPtr) -= tmpW32; + fPtr--; + } + *fPtr -= *lspPtr * (1 << 10); + + fPtr+=i; + lspPtr+=2; + } + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.h new file mode 100644 index 0000000000..70c9c4d4b4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetLspPoly.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GET_LSP_POLY_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GET_LSP_POLY_H_ + +#include + +/*----------------------------------------------------------------* + * Construct the polynomials F1(z) and F2(z) from the LSP + * (Computations are done in Q24) + * + * The expansion is performed using the following recursion: + * + * f[0] = 1; + * tmp = -2.0 * lsp[0]; + * f[1] = tmp; + * for (i=2; i<=5; i++) { + * b = -2.0 * lsp[2*i-2]; + * f[i] = tmp*f[i-1] + 2.0*f[i-2]; + * for (j=i; j>=2; j--) { + * f[j] = f[j] + tmp*f[j-1] + f[j-2]; + * } + * f[i] = f[i] + tmp; + * } + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_GetLspPoly(int16_t* lsp, /* (i) LSP in Q15 */ + int32_t* f); /* (o) polonymial in Q24 */ + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c new file mode 100644 index 0000000000..68a569a40a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetSyncSeq.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/get_sync_seq.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/nearest_neighbor.h" +#include "modules/audio_coding/codecs/ilbc/refiner.h" + +/*----------------------------------------------------------------* + * get the pitch-synchronous sample sequence + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_GetSyncSeq( + int16_t *idata, /* (i) original data */ + size_t idatal, /* (i) dimension of data */ + size_t centerStartPos, /* (i) where current block starts */ + size_t *period, /* (i) rough-pitch-period array (Q-2) */ + const size_t *plocs, /* (i) where periods of period array are taken (Q-2) */ + size_t periodl, /* (i) dimension period array */ + size_t hl, /* (i) 2*hl+1 is the number of sequences */ + int16_t *surround /* (i/o) The contribution from this sequence + summed with earlier contributions */ + ){ + size_t i, centerEndPos, q; + /* Stack based */ + size_t lagBlock[2 * ENH_HL + 1]; + size_t blockStartPos[2 * ENH_HL + 1]; /* The position to search around (Q2) */ + size_t plocs2[ENH_PLOCSL]; + + centerEndPos = centerStartPos + ENH_BLOCKL - 1; + + /* present (find predicted lag from this position) */ + + WebRtcIlbcfix_NearestNeighbor(lagBlock + hl, + plocs, + 2 * (centerStartPos + centerEndPos), + periodl); + + blockStartPos[hl] = 4 * centerStartPos; + + /* past (find predicted position and perform a refined + search to find the best sequence) */ + + for (q = hl; q > 0; q--) { + size_t qq = q - 1; + size_t period_q = period[lagBlock[q]]; + /* Stop if this sequence would be outside the buffer; that means all + further-past sequences would also be outside the buffer. */ + if (blockStartPos[q] < period_q + (4 * ENH_OVERHANG)) + break; + blockStartPos[qq] = blockStartPos[q] - period_q; + + size_t value = blockStartPos[qq] + 4 * ENH_BLOCKL_HALF; + value = (value > period_q) ? (value - period_q) : 0; + WebRtcIlbcfix_NearestNeighbor(lagBlock + qq, plocs, value, periodl); + + /* Find the best possible sequence in the 4 times upsampled + domain around blockStartPos+q */ + WebRtcIlbcfix_Refiner(blockStartPos + qq, idata, idatal, centerStartPos, + blockStartPos[qq], surround, + WebRtcIlbcfix_kEnhWt[qq]); + } + + /* future (find predicted position and perform a refined + search to find the best sequence) */ + + for (i = 0; i < periodl; i++) { + plocs2[i] = plocs[i] - period[i]; + } + + for (q = hl + 1; q <= (2 * hl); q++) { + + WebRtcIlbcfix_NearestNeighbor( + lagBlock + q, + plocs2, + blockStartPos[q - 1] + 4 * ENH_BLOCKL_HALF, + periodl); + + blockStartPos[q]=blockStartPos[q-1]+period[lagBlock[q]]; + + if (blockStartPos[q] + 4 * (ENH_BLOCKL + ENH_OVERHANG) < 4 * idatal) { + + /* Find the best possible sequence in the 4 times upsampled + domain around blockStartPos+q */ + WebRtcIlbcfix_Refiner(blockStartPos + q, idata, idatal, centerStartPos, + blockStartPos[q], surround, + WebRtcIlbcfix_kEnhWt[2 * hl - q]); + + } else { + /* Don't add anything since this sequence would + be outside the buffer */ + } + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.h new file mode 100644 index 0000000000..90962fa063 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_GetSyncSeq.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GET_SYNC_SEQ_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_GET_SYNC_SEQ_H_ + +#include +#include + +/*----------------------------------------------------------------* + * get the pitch-synchronous sample sequence + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_GetSyncSeq( + int16_t* idata, /* (i) original data */ + size_t idatal, /* (i) dimension of data */ + size_t centerStartPos, /* (i) where current block starts */ + size_t* period, /* (i) rough-pitch-period array (Q-2) */ + const size_t* plocs, /* (i) where periods of period array are taken (Q-2) */ + size_t periodl, /* (i) dimension period array */ + size_t hl, /* (i) 2*hl+1 is the number of sequences */ + int16_t* surround /* (i/o) The contribution from this sequence + summed with earlier contributions */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.c new file mode 100644 index 0000000000..be582f2e23 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_HpInput.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/hp_input.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * high-pass filter of input with *0.5 and saturation + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_HpInput( + int16_t *signal, /* (i/o) signal vector */ + int16_t *ba, /* (i) B- and A-coefficients (2:nd order) + {b[0] b[1] b[2] -a[1] -a[2]} a[0] + is assumed to be 1.0 */ + int16_t *y, /* (i/o) Filter state yhi[n-1] ylow[n-1] + yhi[n-2] ylow[n-2] */ + int16_t *x, /* (i/o) Filter state x[n-1] x[n-2] */ + size_t len) /* (i) Number of samples to filter */ +{ + size_t i; + int32_t tmpW32; + int32_t tmpW32b; + + for (i=0; i>15); + tmpW32 += y[0] * ba[3]; /* (-a[1])*y[i-1] (high part) */ + tmpW32 += y[2] * ba[4]; /* (-a[2])*y[i-2] (high part) */ + tmpW32 = (tmpW32<<1); + + tmpW32 += signal[i] * ba[0]; /* b[0]*x[0] */ + tmpW32 += x[0] * ba[1]; /* b[1]*x[i-1] */ + tmpW32 += x[1] * ba[2]; /* b[2]*x[i-2] */ + + /* Update state (input part) */ + x[1] = x[0]; + x[0] = signal[i]; + + /* Rounding in Q(12+1), i.e. add 2^12 */ + tmpW32b = tmpW32 + 4096; + + /* Saturate (to 2^28) so that the HP filtered signal does not overflow */ + tmpW32b = WEBRTC_SPL_SAT((int32_t)268435455, tmpW32b, (int32_t)-268435456); + + /* Convert back to Q0 and multiply with 0.5 */ + signal[i] = (int16_t)(tmpW32b >> 13); + + /* Update state (filtered part) */ + y[2] = y[0]; + y[3] = y[1]; + + /* upshift tmpW32 by 3 with saturation */ + if (tmpW32>268435455) { + tmpW32 = WEBRTC_SPL_WORD32_MAX; + } else if (tmpW32<-268435456) { + tmpW32 = WEBRTC_SPL_WORD32_MIN; + } else { + tmpW32 <<= 3; + } + + y[0] = (int16_t)(tmpW32 >> 16); + y[1] = (int16_t)((tmpW32 - (y[0] << 16)) >> 1); + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.h new file mode 100644 index 0000000000..9143d8efed --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_HpInput.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_HP_INPUT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_HP_INPUT_H_ + +#include +#include + +// clang-format off +// Bad job here. https://bugs.llvm.org/show_bug.cgi?id=34274 +void WebRtcIlbcfix_HpInput( + int16_t* signal, /* (i/o) signal vector */ + int16_t* ba, /* (i) B- and A-coefficients (2:nd order) + {b[0] b[1] b[2] -a[1] -a[2]} + a[0] is assumed to be 1.0 */ + int16_t* y, /* (i/o) Filter state yhi[n-1] ylow[n-1] + yhi[n-2] ylow[n-2] */ + int16_t* x, /* (i/o) Filter state x[n-1] x[n-2] */ + size_t len); /* (i) Number of samples to filter */ +// clang-format on + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.c new file mode 100644 index 0000000000..cc5f6dcd37 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.c @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_HpOutput.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/hp_output.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * high-pass filter of output and *2 with saturation + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_HpOutput( + int16_t *signal, /* (i/o) signal vector */ + int16_t *ba, /* (i) B- and A-coefficients (2:nd order) + {b[0] b[1] b[2] -a[1] -a[2]} a[0] + is assumed to be 1.0 */ + int16_t *y, /* (i/o) Filter state yhi[n-1] ylow[n-1] + yhi[n-2] ylow[n-2] */ + int16_t *x, /* (i/o) Filter state x[n-1] x[n-2] */ + size_t len) /* (i) Number of samples to filter */ +{ + size_t i; + int32_t tmpW32; + int32_t tmpW32b; + + for (i=0; i>15); + tmpW32 += y[0] * ba[3]; /* (-a[1])*y[i-1] (high part) */ + tmpW32 += y[2] * ba[4]; /* (-a[2])*y[i-2] (high part) */ + tmpW32 *= 2; + + tmpW32 += signal[i] * ba[0]; /* b[0]*x[0] */ + tmpW32 += x[0] * ba[1]; /* b[1]*x[i-1] */ + tmpW32 += x[1] * ba[2]; /* b[2]*x[i-2] */ + + /* Update state (input part) */ + x[1] = x[0]; + x[0] = signal[i]; + + /* Rounding in Q(12-1), i.e. add 2^10 */ + tmpW32b = tmpW32 + 1024; + + /* Saturate (to 2^26) so that the HP filtered signal does not overflow */ + tmpW32b = WEBRTC_SPL_SAT((int32_t)67108863, tmpW32b, (int32_t)-67108864); + + /* Convert back to Q0 and multiply with 2 */ + signal[i] = (int16_t)(tmpW32b >> 11); + + /* Update state (filtered part) */ + y[2] = y[0]; + y[3] = y[1]; + + /* upshift tmpW32 by 3 with saturation */ + if (tmpW32>268435455) { + tmpW32 = WEBRTC_SPL_WORD32_MAX; + } else if (tmpW32<-268435456) { + tmpW32 = WEBRTC_SPL_WORD32_MIN; + } else { + tmpW32 *= 8; + } + + y[0] = (int16_t)(tmpW32 >> 16); + y[1] = (int16_t)((tmpW32 & 0xffff) >> 1); + + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.h new file mode 100644 index 0000000000..6d1bd3cd88 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_HpOutput.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_HP_OUTPUT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_HP_OUTPUT_H_ + +#include +#include + +// clang-format off +// Bad job here. https://bugs.llvm.org/show_bug.cgi?id=34274 +void WebRtcIlbcfix_HpOutput( + int16_t* signal, /* (i/o) signal vector */ + int16_t* ba, /* (i) B- and A-coefficients (2:nd order) + {b[0] b[1] b[2] -a[1] -a[2]} a[0] + is assumed to be 1.0 */ + int16_t* y, /* (i/o) Filter state yhi[n-1] ylow[n-1] + yhi[n-2] ylow[n-2] */ + int16_t* x, /* (i/o) Filter state x[n-1] x[n-2] */ + size_t len); /* (i) Number of samples to filter */ +// clang-format on + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.c new file mode 100644 index 0000000000..ba6c3e46c3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.c @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + iLBCInterface.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/ilbc.h" + +#include + +#include "modules/audio_coding/codecs/ilbc/decode.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/encode.h" +#include "modules/audio_coding/codecs/ilbc/init_decode.h" +#include "modules/audio_coding/codecs/ilbc/init_encode.h" +#include "rtc_base/checks.h" + +int16_t WebRtcIlbcfix_EncoderAssign(IlbcEncoderInstance** iLBC_encinst, + int16_t* ILBCENC_inst_Addr, + int16_t* size) { + *iLBC_encinst=(IlbcEncoderInstance*)ILBCENC_inst_Addr; + *size=sizeof(IlbcEncoder)/sizeof(int16_t); + if (*iLBC_encinst!=NULL) { + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcIlbcfix_DecoderAssign(IlbcDecoderInstance** iLBC_decinst, + int16_t* ILBCDEC_inst_Addr, + int16_t* size) { + *iLBC_decinst=(IlbcDecoderInstance*)ILBCDEC_inst_Addr; + *size=sizeof(IlbcDecoder)/sizeof(int16_t); + if (*iLBC_decinst!=NULL) { + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcIlbcfix_EncoderCreate(IlbcEncoderInstance **iLBC_encinst) { + *iLBC_encinst=(IlbcEncoderInstance*)malloc(sizeof(IlbcEncoder)); + if (*iLBC_encinst!=NULL) { + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcIlbcfix_DecoderCreate(IlbcDecoderInstance **iLBC_decinst) { + *iLBC_decinst=(IlbcDecoderInstance*)malloc(sizeof(IlbcDecoder)); + if (*iLBC_decinst!=NULL) { + return(0); + } else { + return(-1); + } +} + +int16_t WebRtcIlbcfix_EncoderFree(IlbcEncoderInstance *iLBC_encinst) { + free(iLBC_encinst); + return(0); +} + +int16_t WebRtcIlbcfix_DecoderFree(IlbcDecoderInstance *iLBC_decinst) { + free(iLBC_decinst); + return(0); +} + +int16_t WebRtcIlbcfix_EncoderInit(IlbcEncoderInstance* iLBCenc_inst, + int16_t mode) { + if ((mode==20)||(mode==30)) { + WebRtcIlbcfix_InitEncode((IlbcEncoder*) iLBCenc_inst, mode); + return(0); + } else { + return(-1); + } +} + +int WebRtcIlbcfix_Encode(IlbcEncoderInstance* iLBCenc_inst, + const int16_t* speechIn, + size_t len, + uint8_t* encoded) { + size_t pos = 0; + size_t encpos = 0; + + if ((len != ((IlbcEncoder*)iLBCenc_inst)->blockl) && +#ifdef SPLIT_10MS + (len != 80) && +#endif + (len != 2*((IlbcEncoder*)iLBCenc_inst)->blockl) && + (len != 3*((IlbcEncoder*)iLBCenc_inst)->blockl)) + { + /* A maximum of 3 frames/packet is allowed */ + return(-1); + } else { + + /* call encoder */ + while (possection == 0) +#else + pos += ((IlbcEncoder*)iLBCenc_inst)->blockl; +#endif + encpos += ((IlbcEncoder*)iLBCenc_inst)->no_of_words; + } + return (int)(encpos*2); + } +} + +int16_t WebRtcIlbcfix_DecoderInit(IlbcDecoderInstance* iLBCdec_inst, + int16_t mode) { + if ((mode==20)||(mode==30)) { + WebRtcIlbcfix_InitDecode((IlbcDecoder*) iLBCdec_inst, mode, 1); + return(0); + } else { + return(-1); + } +} +void WebRtcIlbcfix_DecoderInit20Ms(IlbcDecoderInstance* iLBCdec_inst) { + WebRtcIlbcfix_InitDecode((IlbcDecoder*) iLBCdec_inst, 20, 1); +} +void WebRtcIlbcfix_Decoderinit30Ms(IlbcDecoderInstance* iLBCdec_inst) { + WebRtcIlbcfix_InitDecode((IlbcDecoder*) iLBCdec_inst, 30, 1); +} + + +int WebRtcIlbcfix_Decode(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) +{ + size_t i=0; + /* Allow for automatic switching between the frame sizes + (although you do get some discontinuity) */ + if ((len==((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==2*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==3*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)) { + /* ok, do nothing */ + } else { + /* Test if the mode has changed */ + if (((IlbcDecoder*)iLBCdec_inst)->mode==20) { + if ((len==NO_OF_BYTES_30MS)|| + (len==2*NO_OF_BYTES_30MS)|| + (len==3*NO_OF_BYTES_30MS)) { + WebRtcIlbcfix_InitDecode( + ((IlbcDecoder*)iLBCdec_inst), 30, + ((IlbcDecoder*)iLBCdec_inst)->use_enhancer); + } else { + /* Unsupported frame length */ + return(-1); + } + } else { + if ((len==NO_OF_BYTES_20MS)|| + (len==2*NO_OF_BYTES_20MS)|| + (len==3*NO_OF_BYTES_20MS)) { + WebRtcIlbcfix_InitDecode( + ((IlbcDecoder*)iLBCdec_inst), 20, + ((IlbcDecoder*)iLBCdec_inst)->use_enhancer); + } else { + /* Unsupported frame length */ + return(-1); + } + } + } + + while ((i*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)blockl], + (const uint16_t*)&encoded + [2 * i * ((IlbcDecoder*)iLBCdec_inst)->no_of_words], + (IlbcDecoder*)iLBCdec_inst, 1) == -1) + return -1; + i++; + } + /* iLBC does not support VAD/CNG yet */ + *speechType=1; + return (int)(i*((IlbcDecoder*)iLBCdec_inst)->blockl); +} + +int WebRtcIlbcfix_Decode20Ms(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) +{ + size_t i=0; + if ((len==((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==2*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==3*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)) { + /* ok, do nothing */ + } else { + return(-1); + } + + while ((i*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)blockl], + (const uint16_t*)&encoded + [2 * i * ((IlbcDecoder*)iLBCdec_inst)->no_of_words], + (IlbcDecoder*)iLBCdec_inst, 1)) + return -1; + i++; + } + /* iLBC does not support VAD/CNG yet */ + *speechType=1; + return (int)(i*((IlbcDecoder*)iLBCdec_inst)->blockl); +} + +int WebRtcIlbcfix_Decode30Ms(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType) +{ + size_t i=0; + if ((len==((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==2*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)|| + (len==3*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)) { + /* ok, do nothing */ + } else { + return(-1); + } + + while ((i*((IlbcDecoder*)iLBCdec_inst)->no_of_bytes)blockl], + (const uint16_t*)&encoded + [2 * i * ((IlbcDecoder*)iLBCdec_inst)->no_of_words], + (IlbcDecoder*)iLBCdec_inst, 1)) + return -1; + i++; + } + /* iLBC does not support VAD/CNG yet */ + *speechType=1; + return (int)(i*((IlbcDecoder*)iLBCdec_inst)->blockl); +} + +size_t WebRtcIlbcfix_DecodePlc(IlbcDecoderInstance* iLBCdec_inst, + int16_t* decoded, + size_t noOfLostFrames) { + size_t i; + uint16_t dummy; + + for (i=0;iblockl], &dummy, + (IlbcDecoder*)iLBCdec_inst, 0); + RTC_CHECK_EQ(result, 0); + } + return (noOfLostFrames*((IlbcDecoder*)iLBCdec_inst)->blockl); +} + +size_t WebRtcIlbcfix_NetEqPlc(IlbcDecoderInstance* iLBCdec_inst, + int16_t* decoded, + size_t noOfLostFrames) { + /* Two input parameters not used, but needed for function pointers in NetEQ */ + (void)(decoded = NULL); + (void)(noOfLostFrames = 0); + + WebRtcSpl_MemSetW16(((IlbcDecoder*)iLBCdec_inst)->enh_buf, 0, ENH_BUFL); + ((IlbcDecoder*)iLBCdec_inst)->prev_enh_pl = 2; + + return (0); +} + +void WebRtcIlbcfix_version(char *version) +{ + strcpy((char*)version, "1.1.1"); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.h new file mode 100644 index 0000000000..de8cfde111 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.h @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * ilbc.h + * + * This header file contains all of the API's for iLBC. + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_ILBC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_ILBC_H_ + +#include +#include + +/* + * Solution to support multiple instances + * Customer has to cast instance to proper type + */ + +typedef struct iLBC_encinst_t_ IlbcEncoderInstance; + +typedef struct iLBC_decinst_t_ IlbcDecoderInstance; + +/* + * Comfort noise constants + */ + +#define ILBC_SPEECH 1 +#define ILBC_CNG 2 + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * WebRtcIlbcfix_XxxAssign(...) + * + * These functions assigns the encoder/decoder instance to the specified + * memory location + * + * Input: + * - XXX_xxxinst : Pointer to created instance that should be + * assigned + * - ILBCXXX_inst_Addr : Pointer to the desired memory space + * - size : The size that this structure occupies (in Word16) + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIlbcfix_EncoderAssign(IlbcEncoderInstance** iLBC_encinst, + int16_t* ILBCENC_inst_Addr, + int16_t* size); +int16_t WebRtcIlbcfix_DecoderAssign(IlbcDecoderInstance** iLBC_decinst, + int16_t* ILBCDEC_inst_Addr, + int16_t* size); + +/**************************************************************************** + * WebRtcIlbcfix_XxxAssign(...) + * + * These functions create a instance to the specified structure + * + * Input: + * - XXX_inst : Pointer to created instance that should be created + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIlbcfix_EncoderCreate(IlbcEncoderInstance** iLBC_encinst); +int16_t WebRtcIlbcfix_DecoderCreate(IlbcDecoderInstance** iLBC_decinst); + +/**************************************************************************** + * WebRtcIlbcfix_XxxFree(...) + * + * These functions frees the dynamic memory of a specified instance + * + * Input: + * - XXX_inst : Pointer to created instance that should be freed + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIlbcfix_EncoderFree(IlbcEncoderInstance* iLBC_encinst); +int16_t WebRtcIlbcfix_DecoderFree(IlbcDecoderInstance* iLBC_decinst); + +/**************************************************************************** + * WebRtcIlbcfix_EncoderInit(...) + * + * This function initializes a iLBC instance + * + * Input: + * - iLBCenc_inst : iLBC instance, i.e. the user that should receive + * be initialized + * - frameLen : The frame length of the codec 20/30 (ms) + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIlbcfix_EncoderInit(IlbcEncoderInstance* iLBCenc_inst, + int16_t frameLen); + +/**************************************************************************** + * WebRtcIlbcfix_Encode(...) + * + * This function encodes one iLBC frame. Input speech length has be a + * multiple of the frame length. + * + * Input: + * - iLBCenc_inst : iLBC instance, i.e. the user that should encode + * a package + * - speechIn : Input speech vector + * - len : Samples in speechIn (160, 240, 320 or 480) + * + * Output: + * - encoded : The encoded data vector + * + * Return value : >0 - Length (in bytes) of coded data + * -1 - Error + */ + +int WebRtcIlbcfix_Encode(IlbcEncoderInstance* iLBCenc_inst, + const int16_t* speechIn, + size_t len, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcIlbcfix_DecoderInit(...) + * + * This function initializes a iLBC instance with either 20 or 30 ms frames + * Alternatively the WebRtcIlbcfix_DecoderInit_XXms can be used. Then it's + * not needed to specify the frame length with a variable. + * + * Input: + * - IlbcDecoderInstance : iLBC decoder instance + * - frameLen : The frame length of the codec 20/30 (ms) + * + * Return value : 0 - Ok + * -1 - Error + */ + +int16_t WebRtcIlbcfix_DecoderInit(IlbcDecoderInstance* iLBCdec_inst, + int16_t frameLen); +void WebRtcIlbcfix_DecoderInit20Ms(IlbcDecoderInstance* iLBCdec_inst); +void WebRtcIlbcfix_Decoderinit30Ms(IlbcDecoderInstance* iLBCdec_inst); + +/**************************************************************************** + * WebRtcIlbcfix_Decode(...) + * + * This function decodes a packet with iLBC frame(s). Output speech length + * will be a multiple of 160 or 240 samples ((160 or 240)*frames/packet). + * + * Input: + * - iLBCdec_inst : iLBC instance, i.e. the user that should decode + * a packet + * - encoded : Encoded iLBC frame(s) + * - len : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - speechType : 1 normal, 2 CNG + * + * Return value : >0 - Samples in decoded vector + * -1 - Error + */ + +int WebRtcIlbcfix_Decode(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); +int WebRtcIlbcfix_Decode20Ms(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); +int WebRtcIlbcfix_Decode30Ms(IlbcDecoderInstance* iLBCdec_inst, + const uint8_t* encoded, + size_t len, + int16_t* decoded, + int16_t* speechType); + +/**************************************************************************** + * WebRtcIlbcfix_DecodePlc(...) + * + * This function conducts PLC for iLBC frame(s). Output speech length + * will be a multiple of 160 or 240 samples. + * + * Input: + * - iLBCdec_inst : iLBC instance, i.e. the user that should perform + * a PLC + * - noOfLostFrames : Number of PLC frames to produce + * + * Output: + * - decoded : The "decoded" vector + * + * Return value : Samples in decoded PLC vector + */ + +size_t WebRtcIlbcfix_DecodePlc(IlbcDecoderInstance* iLBCdec_inst, + int16_t* decoded, + size_t noOfLostFrames); + +/**************************************************************************** + * WebRtcIlbcfix_NetEqPlc(...) + * + * This function updates the decoder when a packet loss has occured, but it + * does not produce any PLC data. Function can be used if another PLC method + * is used (i.e NetEq). + * + * Input: + * - iLBCdec_inst : iLBC instance that should be updated + * - noOfLostFrames : Number of lost frames + * + * Output: + * - decoded : The "decoded" vector (nothing in this case) + * + * Return value : Samples in decoded PLC vector + */ + +size_t WebRtcIlbcfix_NetEqPlc(IlbcDecoderInstance* iLBCdec_inst, + int16_t* decoded, + size_t noOfLostFrames); + +/**************************************************************************** + * WebRtcIlbcfix_version(...) + * + * This function returns the version number of iLBC + * + * Output: + * - version : Version number of iLBC (maximum 20 char) + */ + +void WebRtcIlbcfix_version(char* version); + +#ifdef __cplusplus +} +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_ILBC_ILBC_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc new file mode 100644 index 0000000000..689292f131 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc_unittest.cc @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(IlbcTest, BadPacket) { + // Get a good packet. + AudioEncoderIlbcConfig config; + config.frame_size_ms = 20; // We need 20 ms rather than the default 30 ms; + // otherwise, all possible values of cb_index[2] + // are valid. + AudioEncoderIlbcImpl encoder(config, 102); + std::vector samples(encoder.SampleRateHz() / 100, 4711); + rtc::Buffer packet; + int num_10ms_chunks = 0; + while (packet.size() == 0) { + encoder.Encode(0, samples, &packet); + num_10ms_chunks += 1; + } + + // Break the packet by setting all bits of the unsigned 7-bit number + // cb_index[2] to 1, giving it a value of 127. For a 20 ms packet, this is + // too large. + EXPECT_EQ(38u, packet.size()); + rtc::Buffer bad_packet(packet.data(), packet.size()); + bad_packet[29] |= 0x3f; // Bits 1-6. + bad_packet[30] |= 0x80; // Bit 0. + + // Decode the bad packet. We expect the decoder to respond by returning -1. + AudioDecoderIlbcImpl decoder; + std::vector decoded_samples(num_10ms_chunks * samples.size()); + AudioDecoder::SpeechType speech_type; + EXPECT_EQ(-1, decoder.Decode(bad_packet.data(), bad_packet.size(), + encoder.SampleRateHz(), + sizeof(int16_t) * decoded_samples.size(), + decoded_samples.data(), &speech_type)); + + // Decode the good packet. This should work, because the failed decoding + // should not have left the decoder in a broken state. + EXPECT_EQ(static_cast(decoded_samples.size()), + decoder.Decode(packet.data(), packet.size(), encoder.SampleRateHz(), + sizeof(int16_t) * decoded_samples.size(), + decoded_samples.data(), &speech_type)); +} + +class SplitIlbcTest : public ::testing::TestWithParam > { + protected: + virtual void SetUp() { + const std::pair parameters = GetParam(); + num_frames_ = parameters.first; + frame_length_ms_ = parameters.second; + frame_length_bytes_ = (frame_length_ms_ == 20) ? 38 : 50; + } + size_t num_frames_; + int frame_length_ms_; + size_t frame_length_bytes_; +}; + +TEST_P(SplitIlbcTest, NumFrames) { + AudioDecoderIlbcImpl decoder; + const size_t frame_length_samples = frame_length_ms_ * 8; + const auto generate_payload = [](size_t payload_length_bytes) { + rtc::Buffer payload(payload_length_bytes); + // Fill payload with increasing integers {0, 1, 2, ...}. + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i); + } + return payload; + }; + + const auto results = decoder.ParsePayload( + generate_payload(frame_length_bytes_ * num_frames_), 0); + EXPECT_EQ(num_frames_, results.size()); + + size_t frame_num = 0; + uint8_t payload_value = 0; + for (const auto& result : results) { + EXPECT_EQ(frame_length_samples * frame_num, result.timestamp); + const LegacyEncodedAudioFrame* frame = + static_cast(result.frame.get()); + const rtc::Buffer& payload = frame->payload(); + EXPECT_EQ(frame_length_bytes_, payload.size()); + for (size_t i = 0; i < payload.size(); ++i, ++payload_value) { + EXPECT_EQ(payload_value, payload[i]); + } + ++frame_num; + } +} + +// Test 1 through 5 frames of 20 and 30 ms size. +// Also test the maximum number of frames in one packet for 20 and 30 ms. +// The maximum is defined by the largest payload length that can be uniquely +// resolved to a frame size of either 38 bytes (20 ms) or 50 bytes (30 ms). +INSTANTIATE_TEST_SUITE_P( + IlbcTest, + SplitIlbcTest, + ::testing::Values(std::pair(1, 20), // 1 frame, 20 ms. + std::pair(2, 20), // 2 frames, 20 ms. + std::pair(3, 20), // And so on. + std::pair(4, 20), + std::pair(5, 20), + std::pair(24, 20), + std::pair(1, 30), + std::pair(2, 30), + std::pair(3, 30), + std::pair(4, 30), + std::pair(5, 30), + std::pair(18, 30))); + +// Test too large payload size. +TEST(IlbcTest, SplitTooLargePayload) { + AudioDecoderIlbcImpl decoder; + constexpr size_t kPayloadLengthBytes = 950; + const auto results = + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0); + EXPECT_TRUE(results.empty()); +} + +// Payload not an integer number of frames. +TEST(IlbcTest, SplitUnevenPayload) { + AudioDecoderIlbcImpl decoder; + constexpr size_t kPayloadLengthBytes = 39; // Not an even number of frames. + const auto results = + decoder.ParsePayload(rtc::Buffer(kPayloadLengthBytes), 0); + EXPECT_TRUE(results.empty()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c new file mode 100644 index 0000000000..d78f81a897 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_IndexConvDec.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/index_conv_dec.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +void WebRtcIlbcfix_IndexConvDec( + int16_t *index /* (i/o) Codebook indexes */ + ){ + int k; + + for (k=4;k<6;k++) { + /* Readjust the second and third codebook index for the first 40 sample + so that they look the same as the first (in terms of lag) + */ + if ((index[k]>=44)&&(index[k]<108)) { + index[k]+=64; + } else if ((index[k]>=108)&&(index[k]<128)) { + index[k]+=128; + } else { + /* ERROR */ + } + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.h new file mode 100644 index 0000000000..4f08ce04df --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_IndexConvDec.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INDEX_CONV_DEC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INDEX_CONV_DEC_H_ + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +void WebRtcIlbcfix_IndexConvDec(int16_t* index /* (i/o) Codebook indexes */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c new file mode 100644 index 0000000000..83144150b4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + IiLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_IndexConvEnc.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/index_conv_enc.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Convert the codebook indexes to make the search easier + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_IndexConvEnc( + int16_t *index /* (i/o) Codebook indexes */ + ){ + int k; + + for (k=4;k<6;k++) { + /* Readjust the second and third codebook index so that it is + packetized into 7 bits (before it was put in lag-wise the same + way as for the first codebook which uses 8 bits) + */ + if ((index[k]>=108)&&(index[k]<172)) { + index[k]-=64; + } else if (index[k]>=236) { + index[k]-=128; + } else { + /* ERROR */ + } + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.h new file mode 100644 index 0000000000..4fbf98084e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_IndexConvEnc.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INDEX_CONV_ENC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INDEX_CONV_ENC_H_ + +#include + +/*----------------------------------------------------------------* + * Convert the codebook indexes to make the search easier + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_IndexConvEnc(int16_t* index /* (i/o) Codebook indexes */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.c new file mode 100644 index 0000000000..3eb41e33b0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InitDecode.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/init_decode.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Initiation of decoder instance. + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_InitDecode( /* (o) Number of decoded samples */ + IlbcDecoder *iLBCdec_inst, /* (i/o) Decoder instance */ + int16_t mode, /* (i) frame size mode */ + int use_enhancer) { /* (i) 1: use enhancer, 0: no enhancer */ + int i; + + iLBCdec_inst->mode = mode; + + /* Set all the variables that are dependent on the frame size mode */ + if (mode==30) { + iLBCdec_inst->blockl = BLOCKL_30MS; + iLBCdec_inst->nsub = NSUB_30MS; + iLBCdec_inst->nasub = NASUB_30MS; + iLBCdec_inst->lpc_n = LPC_N_30MS; + iLBCdec_inst->no_of_bytes = NO_OF_BYTES_30MS; + iLBCdec_inst->no_of_words = NO_OF_WORDS_30MS; + iLBCdec_inst->state_short_len=STATE_SHORT_LEN_30MS; + } + else if (mode==20) { + iLBCdec_inst->blockl = BLOCKL_20MS; + iLBCdec_inst->nsub = NSUB_20MS; + iLBCdec_inst->nasub = NASUB_20MS; + iLBCdec_inst->lpc_n = LPC_N_20MS; + iLBCdec_inst->no_of_bytes = NO_OF_BYTES_20MS; + iLBCdec_inst->no_of_words = NO_OF_WORDS_20MS; + iLBCdec_inst->state_short_len=STATE_SHORT_LEN_20MS; + } + else { + return(-1); + } + + /* Reset all the previous LSF to mean LSF */ + WEBRTC_SPL_MEMCPY_W16(iLBCdec_inst->lsfdeqold, WebRtcIlbcfix_kLsfMean, LPC_FILTERORDER); + + /* Clear the synthesis filter memory */ + WebRtcSpl_MemSetW16(iLBCdec_inst->syntMem, 0, LPC_FILTERORDER); + + /* Set the old synthesis filter to {1.0 0.0 ... 0.0} */ + WebRtcSpl_MemSetW16(iLBCdec_inst->old_syntdenum, 0, ((LPC_FILTERORDER + 1)*NSUB_MAX)); + for (i=0; iold_syntdenum[i*(LPC_FILTERORDER+1)] = 4096; + } + + /* Clear the variables that are used for the PLC */ + iLBCdec_inst->last_lag = 20; + iLBCdec_inst->consPLICount = 0; + iLBCdec_inst->prevPLI = 0; + iLBCdec_inst->perSquare = 0; + iLBCdec_inst->prevLag = 120; + iLBCdec_inst->prevLpc[0] = 4096; + WebRtcSpl_MemSetW16(iLBCdec_inst->prevLpc+1, 0, LPC_FILTERORDER); + WebRtcSpl_MemSetW16(iLBCdec_inst->prevResidual, 0, BLOCKL_MAX); + + /* Initialize the seed for the random number generator */ + iLBCdec_inst->seed = 777; + + /* Set the filter state of the HP filter to 0 */ + WebRtcSpl_MemSetW16(iLBCdec_inst->hpimemx, 0, 2); + WebRtcSpl_MemSetW16(iLBCdec_inst->hpimemy, 0, 4); + + /* Set the variables that are used in the ehnahcer */ + iLBCdec_inst->use_enhancer = use_enhancer; + WebRtcSpl_MemSetW16(iLBCdec_inst->enh_buf, 0, (ENH_BUFL+ENH_BUFL_FILTEROVERHEAD)); + for (i=0;ienh_period[i]=160; /* Q(-4) */ + } + + iLBCdec_inst->prev_enh_pl = 0; + + return (int)(iLBCdec_inst->blockl); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.h new file mode 100644 index 0000000000..a2b7b91287 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InitDecode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INIT_DECODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INIT_DECODE_H_ + +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Initiation of decoder instance. + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_InitDecode(/* (o) Number of decoded samples */ + IlbcDecoder* + iLBCdec_inst, /* (i/o) Decoder instance */ + int16_t mode, /* (i) frame size mode */ + int use_enhancer /* (i) 1 to use enhancer + 0 to run without enhancer */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.c new file mode 100644 index 0000000000..aa858e94bb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InitEncode.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/init_encode.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Initiation of encoder instance. + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_InitEncode( /* (o) Number of bytes encoded */ + IlbcEncoder *iLBCenc_inst, /* (i/o) Encoder instance */ + int16_t mode) { /* (i) frame size mode */ + iLBCenc_inst->mode = mode; + + /* Set all the variables that are dependent on the frame size mode */ + if (mode==30) { + iLBCenc_inst->blockl = BLOCKL_30MS; + iLBCenc_inst->nsub = NSUB_30MS; + iLBCenc_inst->nasub = NASUB_30MS; + iLBCenc_inst->lpc_n = LPC_N_30MS; + iLBCenc_inst->no_of_bytes = NO_OF_BYTES_30MS; + iLBCenc_inst->no_of_words = NO_OF_WORDS_30MS; + iLBCenc_inst->state_short_len=STATE_SHORT_LEN_30MS; + } + else if (mode==20) { + iLBCenc_inst->blockl = BLOCKL_20MS; + iLBCenc_inst->nsub = NSUB_20MS; + iLBCenc_inst->nasub = NASUB_20MS; + iLBCenc_inst->lpc_n = LPC_N_20MS; + iLBCenc_inst->no_of_bytes = NO_OF_BYTES_20MS; + iLBCenc_inst->no_of_words = NO_OF_WORDS_20MS; + iLBCenc_inst->state_short_len=STATE_SHORT_LEN_20MS; + } + else { + return(-1); + } + + /* Clear the buffers and set the previous LSF and LSP to the mean value */ + WebRtcSpl_MemSetW16(iLBCenc_inst->anaMem, 0, LPC_FILTERORDER); + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->lsfold, WebRtcIlbcfix_kLsfMean, LPC_FILTERORDER); + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->lsfdeqold, WebRtcIlbcfix_kLsfMean, LPC_FILTERORDER); + WebRtcSpl_MemSetW16(iLBCenc_inst->lpc_buffer, 0, LPC_LOOKBACK + BLOCKL_MAX); + + /* Set the filter state of the HP filter to 0 */ + WebRtcSpl_MemSetW16(iLBCenc_inst->hpimemx, 0, 2); + WebRtcSpl_MemSetW16(iLBCenc_inst->hpimemy, 0, 4); + +#ifdef SPLIT_10MS + /*Zeroing the past samples for 10msec Split*/ + WebRtcSpl_MemSetW16(iLBCenc_inst->past_samples,0,160); + iLBCenc_inst->section = 0; +#endif + + return (int)(iLBCenc_inst->no_of_bytes); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.h new file mode 100644 index 0000000000..4ada6a30c8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InitEncode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INIT_ENCODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INIT_ENCODE_H_ + +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * Initiation of encoder instance. + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_InitEncode(/* (o) Number of bytes encoded */ + IlbcEncoder* + iLBCenc_inst, /* (i/o) Encoder instance */ + int16_t mode /* (i) frame size mode */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.c new file mode 100644 index 0000000000..17ed244bd4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Interpolate.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/interpolate.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * interpolation between vectors + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Interpolate( + int16_t *out, /* (o) output vector */ + int16_t *in1, /* (i) first input vector */ + int16_t *in2, /* (i) second input vector */ + int16_t coef, /* (i) weight coefficient in Q14 */ + int16_t length) /* (i) number of sample is vectors */ +{ + int i; + int16_t invcoef; + + /* + Performs the operation out[i] = in[i]*coef + (1-coef)*in2[i] (with rounding) + */ + + invcoef = 16384 - coef; /* 16384 = 1.0 (Q14)*/ + for (i = 0; i < length; i++) { + out[i] = (int16_t)((coef * in1[i] + invcoef * in2[i] + 8192) >> 14); + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.h new file mode 100644 index 0000000000..892082b75c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Interpolate.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INTERPOLATE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INTERPOLATE_H_ + +#include + +/*----------------------------------------------------------------* + * interpolation between vectors + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Interpolate( + int16_t* out, /* (o) output vector */ + int16_t* in1, /* (i) first input vector */ + int16_t* in2, /* (i) second input vector */ + int16_t coef, /* (i) weight coefficient in Q14 */ + int16_t length); /* (i) number of sample is vectors */ + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c new file mode 100644 index 0000000000..6dddd6fb86 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InterpolateSamples.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/interpolate_samples.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +void WebRtcIlbcfix_InterpolateSamples( + int16_t *interpSamples, /* (o) The interpolated samples */ + int16_t *CBmem, /* (i) The CB memory */ + size_t lMem /* (i) Length of the CB memory */ + ) { + int16_t *ppi, *ppo, i, j, temp1, temp2; + int16_t *tmpPtr; + + /* Calculate the 20 vectors of interpolated samples (4 samples each) + that are used in the codebooks for lag 20 to 39 */ + tmpPtr = interpSamples; + for (j=0; j<20; j++) { + temp1 = 0; + temp2 = 3; + ppo = CBmem+lMem-4; + ppi = CBmem+lMem-j-24; + for (i=0; i<4; i++) { + + *tmpPtr++ = (int16_t)((WebRtcIlbcfix_kAlpha[temp2] * *ppo) >> 15) + + (int16_t)((WebRtcIlbcfix_kAlpha[temp1] * *ppi) >> 15); + + ppo++; + ppi++; + temp1++; + temp2--; + } + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.h new file mode 100644 index 0000000000..bc665d7854 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_InterpolateSamples.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INTERPOLATE_SAMPLES_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_INTERPOLATE_SAMPLES_H_ + +#include +#include + +/*----------------------------------------------------------------* + * Construct the interpolated samples for the Augmented CB + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_InterpolateSamples( + int16_t* interpSamples, /* (o) The interpolated samples */ + int16_t* CBmem, /* (i) The CB memory */ + size_t lMem /* (i) Length of the CB memory */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c new file mode 100644 index 0000000000..89f6d29724 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LpcEncode.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/lpc_encode.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/lsf_check.h" +#include "modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h" +#include "modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h" +#include "modules/audio_coding/codecs/ilbc/simple_lsf_quant.h" + +/*----------------------------------------------------------------* + * lpc encoder + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LpcEncode( + int16_t *syntdenum, /* (i/o) synthesis filter coefficients + before/after encoding */ + int16_t *weightdenum, /* (i/o) weighting denumerator coefficients + before/after encoding */ + int16_t *lsf_index, /* (o) lsf quantization index */ + int16_t *data, /* (i) Speech to do LPC analysis on */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ) { + /* Stack based */ + int16_t lsf[LPC_FILTERORDER * LPC_N_MAX]; + int16_t lsfdeq[LPC_FILTERORDER * LPC_N_MAX]; + + /* Calculate LSF's from the input speech */ + WebRtcIlbcfix_SimpleLpcAnalysis(lsf, data, iLBCenc_inst); + + /* Quantize the LSF's */ + WebRtcIlbcfix_SimpleLsfQ(lsfdeq, lsf_index, lsf, iLBCenc_inst->lpc_n); + + /* Stableize the LSF's if needed */ + WebRtcIlbcfix_LsfCheck(lsfdeq, LPC_FILTERORDER, iLBCenc_inst->lpc_n); + + /* Calculate the synthesis and weighting filter coefficients from + the optimal LSF and the dequantized LSF */ + WebRtcIlbcfix_SimpleInterpolateLsf(syntdenum, weightdenum, + lsf, lsfdeq, iLBCenc_inst->lsfold, + iLBCenc_inst->lsfdeqold, LPC_FILTERORDER, iLBCenc_inst); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.h new file mode 100644 index 0000000000..a67b77acbf --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LpcEncode.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LPC_ENCODE_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LPC_ENCODE_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * lpc encoder + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LpcEncode( + int16_t* syntdenum, /* (i/o) synthesis filter coefficients + before/after encoding */ + int16_t* weightdenum, /* (i/o) weighting denumerator coefficients + before/after encoding */ + int16_t* lsf_index, /* (o) lsf quantization index */ + int16_t* data, /* (i) Speech to do LPC analysis on */ + IlbcEncoder* iLBCenc_inst + /* (i/o) the encoder state structure */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.c new file mode 100644 index 0000000000..9f0e19a2d9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LsfCheck.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/lsf_check.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * check for stability of lsf coefficients + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_LsfCheck( + int16_t *lsf, /* LSF parameters */ + int dim, /* dimension of LSF */ + int NoAn) /* No of analysis per frame */ +{ + int k,n,m, Nit=2, change=0,pos; + const int16_t eps=319; /* 0.039 in Q13 (50 Hz)*/ + const int16_t eps2=160; /* eps/2.0 in Q13;*/ + const int16_t maxlsf=25723; /* 3.14; (4000 Hz)*/ + const int16_t minlsf=82; /* 0.01; (0 Hz)*/ + + /* LSF separation check*/ + for (n=0;nmaxlsf) { + lsf[pos]=maxlsf; + change=1; + } + } + } + } + + return change; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.h new file mode 100644 index 0000000000..9ba90a31e6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LsfCheck.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_CHECK_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_CHECK_H_ + +#include + +/*----------------------------------------------------------------* + * check for stability of lsf coefficients + *---------------------------------------------------------------*/ + +int WebRtcIlbcfix_LsfCheck(int16_t* lsf, /* LSF parameters */ + int dim, /* dimension of LSF */ + int NoAn); /* No of analysis per frame */ + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c new file mode 100644 index 0000000000..04de5e7e6c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LspInterpolate2PolyDec.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/interpolate.h" +#include "modules/audio_coding/codecs/ilbc/lsf_to_poly.h" + +/*----------------------------------------------------------------* + * interpolation of lsf coefficients for the decoder + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LspInterpolate2PolyDec( + int16_t *a, /* (o) lpc coefficients Q12 */ + int16_t *lsf1, /* (i) first set of lsf coefficients Q13 */ + int16_t *lsf2, /* (i) second set of lsf coefficients Q13 */ + int16_t coef, /* (i) weighting coefficient to use between + lsf1 and lsf2 Q14 */ + int16_t length /* (i) length of coefficient vectors */ + ){ + int16_t lsftmp[LPC_FILTERORDER]; + + /* interpolate LSF */ + WebRtcIlbcfix_Interpolate(lsftmp, lsf1, lsf2, coef, length); + + /* Compute the filter coefficients from the LSF */ + WebRtcIlbcfix_Lsf2Poly(a, lsftmp); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h new file mode 100644 index 0000000000..6cc9d9746d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LspInterpolate2PolyDec.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_INTERPOLATE_TO_POLY_DEC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_INTERPOLATE_TO_POLY_DEC_H_ + +#include + +/*----------------------------------------------------------------* + * interpolation of lsf coefficients for the decoder + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LspInterpolate2PolyDec( + int16_t* a, /* (o) lpc coefficients Q12 */ + int16_t* lsf1, /* (i) first set of lsf coefficients Q13 */ + int16_t* lsf2, /* (i) second set of lsf coefficients Q13 */ + int16_t coef, /* (i) weighting coefficient to use between + lsf1 and lsf2 Q14 */ + int16_t length /* (i) length of coefficient vectors */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c new file mode 100644 index 0000000000..618821216c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LsfInterpolate2PloyEnc.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/interpolate.h" +#include "modules/audio_coding/codecs/ilbc/lsf_to_poly.h" + +/*----------------------------------------------------------------* + * lsf interpolator and conversion from lsf to a coefficients + * (subrutine to SimpleInterpolateLSF) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LsfInterpolate2PloyEnc( + int16_t *a, /* (o) lpc coefficients Q12 */ + int16_t *lsf1, /* (i) first set of lsf coefficients Q13 */ + int16_t *lsf2, /* (i) second set of lsf coefficients Q13 */ + int16_t coef, /* (i) weighting coefficient to use between + lsf1 and lsf2 Q14 */ + int16_t length /* (i) length of coefficient vectors */ + ) { + /* Stack based */ + int16_t lsftmp[LPC_FILTERORDER]; + + /* interpolate LSF */ + WebRtcIlbcfix_Interpolate(lsftmp, lsf1, lsf2, coef, length); + + /* Compute the filter coefficients from the LSF */ + WebRtcIlbcfix_Lsf2Poly(a, lsftmp); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h new file mode 100644 index 0000000000..b278a10f4b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_LsfInterpolate2PloyEnc.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_INTERPOLATE_TO_POLY_ENC_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_INTERPOLATE_TO_POLY_ENC_H_ + +#include + +/*----------------------------------------------------------------* + * lsf interpolator and conversion from lsf to a coefficients + * (subrutine to SimpleInterpolateLSF) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_LsfInterpolate2PloyEnc( + int16_t* a, /* (o) lpc coefficients Q12 */ + int16_t* lsf1, /* (i) first set of lsf coefficients Q13 */ + int16_t* lsf2, /* (i) second set of lsf coefficients Q13 */ + int16_t coef, /* (i) weighting coefficient to use between + lsf1 and lsf2 Q14 */ + int16_t length /* (i) length of coefficient vectors */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c new file mode 100644 index 0000000000..ee8292f394 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsf2Lsp.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/lsf_to_lsp.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * conversion from lsf to lsp coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsf2Lsp( + int16_t *lsf, /* (i) lsf in Q13 values between 0 and pi */ + int16_t *lsp, /* (o) lsp in Q15 values between -1 and 1 */ + int16_t m /* (i) number of coefficients */ + ) { + int16_t i, k; + int16_t diff; /* difference, which is used for the + linear approximation (Q8) */ + int16_t freq; /* normalized frequency in Q15 (0..1) */ + int32_t tmpW32; + + for(i=0; i> 15); + /* 20861: 1.0/(2.0*PI) in Q17 */ + /* + Upper 8 bits give the index k and + Lower 8 bits give the difference, which needs + to be approximated linearly + */ + k = freq >> 8; + diff = (freq&0x00ff); + + /* Guard against getting outside table */ + + if (k>63) { + k = 63; + } + + /* Calculate linear approximation */ + tmpW32 = WebRtcIlbcfix_kCosDerivative[k] * diff; + lsp[i] = WebRtcIlbcfix_kCos[k] + (int16_t)(tmpW32 >> 12); + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.h new file mode 100644 index 0000000000..6bc6c44dbd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsf2Lsp.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_TO_LSP_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_TO_LSP_H_ + +#include + +/*----------------------------------------------------------------* + * conversion from lsf to lsp coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsf2Lsp( + int16_t* lsf, /* (i) lsf in Q13 values between 0 and pi */ + int16_t* lsp, /* (o) lsp in Q15 values between -1 and 1 */ + int16_t m /* (i) number of coefficients */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c new file mode 100644 index 0000000000..8ca91d82f8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsf2Poly.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/lsf_to_poly.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/get_lsp_poly.h" +#include "modules/audio_coding/codecs/ilbc/lsf_to_lsp.h" + +void WebRtcIlbcfix_Lsf2Poly( + int16_t *a, /* (o) predictor coefficients (order = 10) in Q12 */ + int16_t *lsf /* (i) line spectral frequencies in Q13 */ + ) { + int32_t f[2][6]; /* f[0][] and f[1][] corresponds to + F1(z) and F2(z) respectivly */ + int32_t *f1ptr, *f2ptr; + int16_t *a1ptr, *a2ptr; + int32_t tmpW32; + int16_t lsp[10]; + int i; + + /* Convert lsf to lsp */ + WebRtcIlbcfix_Lsf2Lsp(lsf, lsp, LPC_FILTERORDER); + + /* Get F1(z) and F2(z) from the lsp */ + f1ptr=f[0]; + f2ptr=f[1]; + WebRtcIlbcfix_GetLspPoly(&lsp[0],f1ptr); + WebRtcIlbcfix_GetLspPoly(&lsp[1],f2ptr); + + /* for i = 5 down to 1 + Compute f1[i] += f1[i-1]; + and f2[i] += f2[i-1]; + */ + f1ptr=&f[0][5]; + f2ptr=&f[1][5]; + for (i=5; i>0; i--) + { + (*f1ptr) += (*(f1ptr-1)); + (*f2ptr) -= (*(f2ptr-1)); + f1ptr--; + f2ptr--; + } + + /* Get the A(z) coefficients + a[0] = 1.0 + for i = 1 to 5 + a[i] = (f1[i] + f2[i] + round)>>13; + for i = 1 to 5 + a[11-i] = (f1[i] - f2[i] + round)>>13; + */ + a[0]=4096; + a1ptr=&a[1]; + a2ptr=&a[10]; + f1ptr=&f[0][1]; + f2ptr=&f[1][1]; + for (i=5; i>0; i--) + { + tmpW32 = (*f1ptr) + (*f2ptr); + *a1ptr = (int16_t)((tmpW32 + 4096) >> 13); + + tmpW32 = (*f1ptr) - (*f2ptr); + *a2ptr = (int16_t)((tmpW32 + 4096) >> 13); + + a1ptr++; + a2ptr--; + f1ptr++; + f2ptr++; + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.h new file mode 100644 index 0000000000..f26d3a8d2d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsf2Poly.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_TO_POLY_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSF_TO_POLY_H_ + +#include + +/*----------------------------------------------------------------* + * Convert from LSF coefficients to A coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsf2Poly( + int16_t* a, /* (o) predictor coefficients (order = 10) in Q12 */ + int16_t* lsf /* (i) line spectral frequencies in Q13 */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c new file mode 100644 index 0000000000..227f4d45b4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsp2Lsf.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/lsp_to_lsf.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * conversion from LSP coefficients to LSF coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsp2Lsf( + int16_t *lsp, /* (i) lsp vector -1...+1 in Q15 */ + int16_t *lsf, /* (o) Lsf vector 0...Pi in Q13 + (ordered, so that lsf[i]=0; i--) + { + /* + locate value in the table, which is just above lsp[i], + basically an approximation to acos(x) + */ + while( (((int32_t)(*cosTblPtr)-(*lspPtr)) < 0)&&(k>0) ) + { + k-=1; + cosTblPtr--; + } + + /* Calculate diff, which is used in the linear approximation of acos(x) */ + diff = (*lspPtr)-(*cosTblPtr); + + /* + The linear approximation of acos(lsp[i]) : + acos(lsp[i])= k*512 + (WebRtcIlbcfix_kAcosDerivative[ind]*offset >> 11) + */ + + /* tmp (linear offset) in Q16 */ + tmp = (int16_t)((WebRtcIlbcfix_kAcosDerivative[k] * diff) >> 11); + + /* freq in Q16 */ + freq = (k << 9) + tmp; + + /* lsf = freq*2*pi */ + (*lsfPtr) = (int16_t)(((int32_t)freq*25736)>>15); + + lsfPtr--; + lspPtr--; + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.h new file mode 100644 index 0000000000..c2f4b7692d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Lsp2Lsf.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSP_TO_LSF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_LSP_TO_LSF_H_ + +#include + +/*----------------------------------------------------------------* + * conversion from LSP coefficients to LSF coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Lsp2Lsf( + int16_t* lsp, /* (i) lsp vector -1...+1 in Q15 */ + int16_t* lsf, /* (o) Lsf vector 0...Pi in Q13 + (ordered, so that lsf[i] +#include + +/*----------------------------------------------------------------* + * compute cross correlation between sequences + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_MyCorr(int32_t* corr, /* (o) correlation of seq1 and seq2 */ + const int16_t* seq1, /* (i) first sequence */ + size_t dim1, /* (i) dimension first seq1 */ + const int16_t* seq2, /* (i) second sequence */ + size_t dim2 /* (i) dimension seq2 */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.c new file mode 100644 index 0000000000..1ecdd96d5a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_NearestNeighbor.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/nearest_neighbor.h" + +void WebRtcIlbcfix_NearestNeighbor(size_t* index, + const size_t* array, + size_t value, + size_t arlength) { + size_t i; + size_t min_diff = (size_t)-1; + for (i = 0; i < arlength; i++) { + const size_t diff = + (array[i] < value) ? (value - array[i]) : (array[i] - value); + if (diff < min_diff) { + *index = i; + min_diff = diff; + } + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.h new file mode 100644 index 0000000000..704cf2a37d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_NearestNeighbor.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_NEAREST_NEIGHBOR_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_NEAREST_NEIGHBOR_H_ + +#include +#include + +/*----------------------------------------------------------------* + * Find index in array such that the array element with said + * index is the element of said array closest to "value" + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_NearestNeighbor( + size_t* index, /* (o) index of array element closest to value */ + const size_t* array, /* (i) data array (Q2) */ + size_t value, /* (i) value (Q2) */ + size_t arlength /* (i) dimension of data array (==ENH_NBLOCKS_TOT) */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.c new file mode 100644 index 0000000000..dd44eb8fb6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.c @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_PackBits.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/pack_bits.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * unpacking of bits from bitstream, i.e., vector of bytes + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_PackBits( + uint16_t *bitstream, /* (o) The packetized bitstream */ + iLBC_bits *enc_bits, /* (i) Encoded bits */ + int16_t mode /* (i) Codec mode (20 or 30) */ + ){ + uint16_t *bitstreamPtr; + int i, k; + int16_t *tmpPtr; + + bitstreamPtr=bitstream; + + /* Class 1 bits of ULP */ + /* First int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->lsf[0])<<10; /* Bit 0..5 */ + (*bitstreamPtr) |= (enc_bits->lsf[1])<<3; /* Bit 6..12 */ + (*bitstreamPtr) |= (enc_bits->lsf[2]&0x70)>>4; /* Bit 13..15 */ + bitstreamPtr++; + /* Second int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->lsf[2]&0xF)<<12; /* Bit 0..3 */ + + if (mode==20) { + (*bitstreamPtr) |= (enc_bits->startIdx)<<10; /* Bit 4..5 */ + (*bitstreamPtr) |= (enc_bits->state_first)<<9; /* Bit 6 */ + (*bitstreamPtr) |= (enc_bits->idxForMax)<<3; /* Bit 7..12 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[0])&0x70)>>4; /* Bit 13..15 */ + bitstreamPtr++; + /* Third int16_t */ + (*bitstreamPtr) = ((enc_bits->cb_index[0])&0xE)<<12; /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[0])&0x18)<<8; /* Bit 3..4 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[1])&0x8)<<7; /* Bit 5 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[3])&0xFE)<<2; /* Bit 6..12 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[3])&0x10)>>2; /* Bit 13 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[4])&0x8)>>2; /* Bit 14 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[6])&0x10)>>4; /* Bit 15 */ + } else { /* mode==30 */ + (*bitstreamPtr) |= (enc_bits->lsf[3])<<6; /* Bit 4..9 */ + (*bitstreamPtr) |= (enc_bits->lsf[4]&0x7E)>>1; /* Bit 10..15 */ + bitstreamPtr++; + /* Third int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->lsf[4]&0x1)<<15; /* Bit 0 */ + (*bitstreamPtr) |= (enc_bits->lsf[5])<<8; /* Bit 1..7 */ + (*bitstreamPtr) |= (enc_bits->startIdx)<<5; /* Bit 8..10 */ + (*bitstreamPtr) |= (enc_bits->state_first)<<4; /* Bit 11 */ + (*bitstreamPtr) |= ((enc_bits->idxForMax)&0x3C)>>2; /* Bit 12..15 */ + bitstreamPtr++; + /* 4:th int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->idxForMax&0x3)<<14; /* Bit 0..1 */ + (*bitstreamPtr) |= (enc_bits->cb_index[0]&0x78)<<7; /* Bit 2..5 */ + (*bitstreamPtr) |= (enc_bits->gain_index[0]&0x10)<<5; /* Bit 6 */ + (*bitstreamPtr) |= (enc_bits->gain_index[1]&0x8)<<5; /* Bit 7 */ + (*bitstreamPtr) |= (enc_bits->cb_index[3]&0xFC); /* Bit 8..13 */ + (*bitstreamPtr) |= (enc_bits->gain_index[3]&0x10)>>3; /* Bit 14 */ + (*bitstreamPtr) |= (enc_bits->gain_index[4]&0x8)>>3; /* Bit 15 */ + } + /* Class 2 bits of ULP */ + /* 4:th to 6:th int16_t for 20 ms case + 5:th to 7:th int16_t for 30 ms case */ + bitstreamPtr++; + tmpPtr=enc_bits->idxVec; + for (k=0; k<3; k++) { + (*bitstreamPtr) = 0; + for (i=15; i>=0; i--) { + (*bitstreamPtr) |= ((uint16_t)((*tmpPtr)&0x4)>>2)<6; i--) { + (*bitstreamPtr) |= ((uint16_t)((*tmpPtr)&0x4)>>2)<gain_index[1]&0x4)<<4; /* Bit 9 */ + (*bitstreamPtr) |= (enc_bits->gain_index[3]&0xC)<<2; /* Bit 10..11 */ + (*bitstreamPtr) |= (enc_bits->gain_index[4]&0x4)<<1; /* Bit 12 */ + (*bitstreamPtr) |= (enc_bits->gain_index[6]&0x8)>>1; /* Bit 13 */ + (*bitstreamPtr) |= (enc_bits->gain_index[7]&0xC)>>2; /* Bit 14..15 */ + + } else { /* mode==30 */ + /* 8:th int16_t */ + (*bitstreamPtr) = 0; + for (i=15; i>5; i--) { + (*bitstreamPtr) |= ((uint16_t)((*tmpPtr)&0x4)>>2)<cb_index[0]&0x6)<<3; /* Bit 10..11 */ + (*bitstreamPtr) |= (enc_bits->gain_index[0]&0x8); /* Bit 12 */ + (*bitstreamPtr) |= (enc_bits->gain_index[1]&0x4); /* Bit 13 */ + (*bitstreamPtr) |= (enc_bits->cb_index[3]&0x2); /* Bit 14 */ + (*bitstreamPtr) |= (enc_bits->cb_index[6]&0x80)>>7; /* Bit 15 */ + bitstreamPtr++; + /* 9:th int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->cb_index[6]&0x7E)<<9;/* Bit 0..5 */ + (*bitstreamPtr) |= (enc_bits->cb_index[9]&0xFE)<<2; /* Bit 6..12 */ + (*bitstreamPtr) |= (enc_bits->cb_index[12]&0xE0)>>5; /* Bit 13..15 */ + bitstreamPtr++; + /* 10:th int16_t */ + (*bitstreamPtr) = ((uint16_t)enc_bits->cb_index[12]&0x1E)<<11;/* Bit 0..3 */ + (*bitstreamPtr) |= (enc_bits->gain_index[3]&0xC)<<8; /* Bit 4..5 */ + (*bitstreamPtr) |= (enc_bits->gain_index[4]&0x6)<<7; /* Bit 6..7 */ + (*bitstreamPtr) |= (enc_bits->gain_index[6]&0x18)<<3; /* Bit 8..9 */ + (*bitstreamPtr) |= (enc_bits->gain_index[7]&0xC)<<2; /* Bit 10..11 */ + (*bitstreamPtr) |= (enc_bits->gain_index[9]&0x10)>>1; /* Bit 12 */ + (*bitstreamPtr) |= (enc_bits->gain_index[10]&0x8)>>1; /* Bit 13 */ + (*bitstreamPtr) |= (enc_bits->gain_index[12]&0x10)>>3; /* Bit 14 */ + (*bitstreamPtr) |= (enc_bits->gain_index[13]&0x8)>>3; /* Bit 15 */ + } + bitstreamPtr++; + /* Class 3 bits of ULP */ + /* 8:th to 14:th int16_t for 20 ms case + 11:th to 17:th int16_t for 30 ms case */ + tmpPtr=enc_bits->idxVec; + for (k=0; k<7; k++) { + (*bitstreamPtr) = 0; + for (i=14; i>=0; i-=2) { + (*bitstreamPtr) |= ((uint16_t)((*tmpPtr)&0x3))<idxVec[56])&0x3))<<14;/* Bit 0..1 */ + (*bitstreamPtr) |= (((enc_bits->cb_index[0])&1))<<13; /* Bit 2 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[1]))<<6; /* Bit 3..9 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[2])&0x7E)>>1; /* Bit 10..15 */ + bitstreamPtr++; + /* 16:th int16_t */ + (*bitstreamPtr) = ((uint16_t)((enc_bits->cb_index[2])&0x1))<<15; + /* Bit 0 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[0])&0x7)<<12; /* Bit 1..3 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[1])&0x3)<<10; /* Bit 4..5 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[2]))<<7; /* Bit 6..8 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[3])&0x1)<<6; /* Bit 9 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[4])&0x7E)>>1; /* Bit 10..15 */ + bitstreamPtr++; + /* 17:th int16_t */ + (*bitstreamPtr) = ((uint16_t)((enc_bits->cb_index[4])&0x1))<<15; + /* Bit 0 */ + (*bitstreamPtr) |= (enc_bits->cb_index[5])<<8; /* Bit 1..7 */ + (*bitstreamPtr) |= (enc_bits->cb_index[6]); /* Bit 8..15 */ + bitstreamPtr++; + /* 18:th int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[7]))<<8; /* Bit 0..7 */ + (*bitstreamPtr) |= (enc_bits->cb_index[8]); /* Bit 8..15 */ + bitstreamPtr++; + /* 19:th int16_t */ + (*bitstreamPtr) = ((uint16_t)((enc_bits->gain_index[3])&0x3))<<14; + /* Bit 0..1 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[4])&0x3)<<12; /* Bit 2..3 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[5]))<<9; /* Bit 4..6 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[6])&0x7)<<6; /* Bit 7..9 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[7])&0x3)<<4; /* Bit 10..11 */ + (*bitstreamPtr) |= (enc_bits->gain_index[8])<<1; /* Bit 12..14 */ + } else { /* mode==30 */ + /* 18:th int16_t */ + (*bitstreamPtr) = ((uint16_t)((enc_bits->idxVec[56])&0x3))<<14;/* Bit 0..1 */ + (*bitstreamPtr) |= (((enc_bits->idxVec[57])&0x3))<<12; /* Bit 2..3 */ + (*bitstreamPtr) |= (((enc_bits->cb_index[0])&1))<<11; /* Bit 4 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[1]))<<4; /* Bit 5..11 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[2])&0x78)>>3; /* Bit 12..15 */ + bitstreamPtr++; + /* 19:th int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[2])&0x7)<<13; + /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[0])&0x7)<<10; /* Bit 3..5 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[1])&0x3)<<8; /* Bit 6..7 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[2])&0x7)<<5; /* Bit 8..10 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[3])&0x1)<<4; /* Bit 11 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[4])&0x78)>>3; /* Bit 12..15 */ + bitstreamPtr++; + /* 20:th int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[4])&0x7)<<13; + /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[5]))<<6; /* Bit 3..9 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[6])&0x1)<<5; /* Bit 10 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[7])&0xF8)>>3; /* Bit 11..15 */ + bitstreamPtr++; + /* 21:st int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[7])&0x7)<<13; + /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[8]))<<5; /* Bit 3..10 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[9])&0x1)<<4; /* Bit 11 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[10])&0xF0)>>4; /* Bit 12..15 */ + bitstreamPtr++; + /* 22:nd int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[10])&0xF)<<12; + /* Bit 0..3 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[11]))<<4; /* Bit 4..11 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[12])&0x1)<<3; /* Bit 12 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[13])&0xE0)>>5; /* Bit 13..15 */ + bitstreamPtr++; + /* 23:rd int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->cb_index[13])&0x1F)<<11; + /* Bit 0..4 */ + (*bitstreamPtr) |= ((enc_bits->cb_index[14]))<<3; /* Bit 5..12 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[3])&0x3)<<1; /* Bit 13..14 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[4])&0x1); /* Bit 15 */ + bitstreamPtr++; + /* 24:rd int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->gain_index[5]))<<13; + /* Bit 0..2 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[6])&0x7)<<10; /* Bit 3..5 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[7])&0x3)<<8; /* Bit 6..7 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[8]))<<5; /* Bit 8..10 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[9])&0xF)<<1; /* Bit 11..14 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[10])&0x4)>>2; /* Bit 15 */ + bitstreamPtr++; + /* 25:rd int16_t */ + (*bitstreamPtr) = ((uint16_t)(enc_bits->gain_index[10])&0x3)<<14; + /* Bit 0..1 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[11]))<<11; /* Bit 2..4 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[12])&0xF)<<7; /* Bit 5..8 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[13])&0x7)<<4; /* Bit 9..11 */ + (*bitstreamPtr) |= ((enc_bits->gain_index[14]))<<1; /* Bit 12..14 */ + } + /* Last bit is automatically zero */ + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.h new file mode 100644 index 0000000000..8dcf41ce08 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_PackBits.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_PACK_BITS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_PACK_BITS_H_ + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * unpacking of bits from bitstream, i.e., vector of bytes + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_PackBits( + uint16_t* bitstream, /* (o) The packetized bitstream */ + iLBC_bits* enc_bits, /* (i) Encoded bits */ + int16_t mode /* (i) Codec mode (20 or 30) */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c new file mode 100644 index 0000000000..7192eaab49 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Poly2Lsf.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/poly_to_lsf.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/lsp_to_lsf.h" +#include "modules/audio_coding/codecs/ilbc/poly_to_lsp.h" + +void WebRtcIlbcfix_Poly2Lsf( + int16_t *lsf, /* (o) lsf coefficients (Q13) */ + int16_t *a /* (i) A coefficients (Q12) */ + ) { + int16_t lsp[10]; + WebRtcIlbcfix_Poly2Lsp(a, lsp, (int16_t*)WebRtcIlbcfix_kLspMean); + WebRtcIlbcfix_Lsp2Lsf(lsp, lsf, 10); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.h new file mode 100644 index 0000000000..363e392bb2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Poly2Lsf.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_POLY_TO_LSF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_POLY_TO_LSF_H_ + +#include + +/*----------------------------------------------------------------* + * conversion from lpc coefficients to lsf coefficients + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Poly2Lsf(int16_t* lsf, /* (o) lsf coefficients (Q13) */ + int16_t* a /* (i) A coefficients (Q12) */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c new file mode 100644 index 0000000000..ad0ecd70ab --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Poly2Lsp.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/poly_to_lsp.h" + +#include "modules/audio_coding/codecs/ilbc/chebyshev.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" + +/*----------------------------------------------------------------* + * conversion from lpc coefficients to lsp coefficients + * function is only for 10:th order LPC + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Poly2Lsp( + int16_t *a, /* (o) A coefficients in Q12 */ + int16_t *lsp, /* (i) LSP coefficients in Q15 */ + int16_t *old_lsp /* (i) old LSP coefficients that are used if the new + coefficients turn out to be unstable */ + ) { + int16_t f[2][6]; /* f[0][] represents f1 and f[1][] represents f2 */ + int16_t *a_i_ptr, *a_10mi_ptr; + int16_t *f1ptr, *f2ptr; + int32_t tmpW32; + int16_t x, y, xlow, ylow, xmid, ymid, xhigh, yhigh, xint; + int16_t shifts, sign; + int i, j; + int foundFreqs; + int fi_select; + + /* + Calculate the two polynomials f1(z) and f2(z) + (the sum and the diff polynomial) + f1[0] = f2[0] = 1.0; + f1[i+1] = a[i+1] + a[10-i] - f1[i]; + f2[i+1] = a[i+1] - a[10-i] - f1[i]; + */ + + a_i_ptr = a + 1; + a_10mi_ptr = a + 10; + f1ptr = f[0]; + f2ptr = f[1]; + (*f1ptr) = 1024; /* 1.0 in Q10 */ + (*f2ptr) = 1024; /* 1.0 in Q10 */ + for (i = 0; i < 5; i++) { + *(f1ptr + 1) = + (int16_t)((((int32_t)(*a_i_ptr) + *a_10mi_ptr) >> 2) - *f1ptr); + *(f2ptr + 1) = + (int16_t)((((int32_t)(*a_i_ptr) - *a_10mi_ptr) >> 2) + *f2ptr); + a_i_ptr++; + a_10mi_ptr--; + f1ptr++; + f2ptr++; + } + + /* + find the LSPs using the Chebychev pol. evaluation + */ + + fi_select = 0; /* selector between f1 and f2, start with f1 */ + + foundFreqs = 0; + + xlow = WebRtcIlbcfix_kCosGrid[0]; + ylow = WebRtcIlbcfix_Chebyshev(xlow, f[fi_select]); + + /* + Iterate until all the 10 LSP's have been found or + all the grid points have been tried. If the 10 LSP's can + not be found, set the LSP vector to previous LSP + */ + + for (j = 1; j < COS_GRID_POINTS && foundFreqs < 10; j++) { + xhigh = xlow; + yhigh = ylow; + xlow = WebRtcIlbcfix_kCosGrid[j]; + ylow = WebRtcIlbcfix_Chebyshev(xlow, f[fi_select]); + + if (ylow * yhigh <= 0) { + /* Run 4 times to reduce the interval */ + for (i = 0; i < 4; i++) { + /* xmid =(xlow + xhigh)/2 */ + xmid = (xlow >> 1) + (xhigh >> 1); + ymid = WebRtcIlbcfix_Chebyshev(xmid, f[fi_select]); + + if (ylow * ymid <= 0) { + yhigh = ymid; + xhigh = xmid; + } else { + ylow = ymid; + xlow = xmid; + } + } + + /* + Calculater xint by linear interpolation: + xint = xlow - ylow*(xhigh-xlow)/(yhigh-ylow); + */ + + x = xhigh - xlow; + y = yhigh - ylow; + + if (y == 0) { + xint = xlow; + } else { + sign = y; + y = WEBRTC_SPL_ABS_W16(y); + shifts = (int16_t)WebRtcSpl_NormW32(y)-16; + y <<= shifts; + y = (int16_t)WebRtcSpl_DivW32W16(536838144, y); /* 1/(yhigh-ylow) */ + + tmpW32 = (x * y) >> (19 - shifts); + + /* y=(xhigh-xlow)/(yhigh-ylow) */ + y = (int16_t)(tmpW32&0xFFFF); + + if (sign < 0) { + y = -y; + } + /* tmpW32 = ylow*(xhigh-xlow)/(yhigh-ylow) */ + tmpW32 = (ylow * y) >> 10; + xint = xlow-(int16_t)(tmpW32&0xFFFF); + } + + /* Store the calculated lsp */ + lsp[foundFreqs] = (int16_t)xint; + foundFreqs++; + + /* if needed, set xlow and ylow for next recursion */ + if (foundFreqs<10) { + xlow = xint; + /* Swap between f1 and f2 (f[0][] and f[1][]) */ + fi_select = ((fi_select+1)&0x1); + + ylow = WebRtcIlbcfix_Chebyshev(xlow, f[fi_select]); + } + } + } + + /* Check if M roots found, if not then use the old LSP */ + if (foundFreqs < 10) { + WEBRTC_SPL_MEMCPY_W16(lsp, old_lsp, 10); + } + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.h new file mode 100644 index 0000000000..928ee4efdb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Poly2Lsp.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_POLY_TO_LSP_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_POLY_TO_LSP_H_ + +#include + +/*----------------------------------------------------------------* + * conversion from lpc coefficients to lsp coefficients + * function is only for 10:th order LPC + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Poly2Lsp( + int16_t* a, /* (o) A coefficients in Q12 */ + int16_t* lsp, /* (i) LSP coefficients in Q15 */ + int16_t* old_lsp /* (i) old LSP coefficients that are used if the new + coefficients turn out to be unstable */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.c new file mode 100644 index 0000000000..5bdab7a4b0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Refiner.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/refiner.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/enh_upsample.h" +#include "modules/audio_coding/codecs/ilbc/my_corr.h" + +/*----------------------------------------------------------------* + * find segment starting near idata+estSegPos that has highest + * correlation with idata+centerStartPos through + * idata+centerStartPos+ENH_BLOCKL-1 segment is found at a + * resolution of ENH_UPSO times the original of the original + * sampling rate + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Refiner( + size_t *updStartPos, /* (o) updated start point (Q-2) */ + int16_t *idata, /* (i) original data buffer */ + size_t idatal, /* (i) dimension of idata */ + size_t centerStartPos, /* (i) beginning center segment */ + size_t estSegPos, /* (i) estimated beginning other segment (Q-2) */ + int16_t *surround, /* (i/o) The contribution from this sequence + summed with earlier contributions */ + int16_t gain /* (i) Gain to use for this sequence */ + ){ + size_t estSegPosRounded, searchSegStartPos, searchSegEndPos, corrdim; + size_t tloc, tloc2, i; + + int32_t maxtemp, scalefact; + int16_t *filtStatePtr, *polyPtr; + /* Stack based */ + int16_t filt[7]; + int32_t corrVecUps[ENH_CORRDIM*ENH_UPS0]; + int32_t corrVecTemp[ENH_CORRDIM]; + int16_t vect[ENH_VECTL]; + int16_t corrVec[ENH_CORRDIM]; + + /* defining array bounds */ + + estSegPosRounded = (estSegPos - 2) >> 2; + + searchSegStartPos = + (estSegPosRounded < ENH_SLOP) ? 0 : (estSegPosRounded - ENH_SLOP); + + searchSegEndPos = estSegPosRounded + ENH_SLOP; + if ((searchSegEndPos + ENH_BLOCKL) >= idatal) { + searchSegEndPos = idatal - ENH_BLOCKL - 1; + } + + corrdim = searchSegEndPos + 1 - searchSegStartPos; + + /* compute upsampled correlation and find + location of max */ + + WebRtcIlbcfix_MyCorr(corrVecTemp, idata + searchSegStartPos, + corrdim + ENH_BLOCKL - 1, idata + centerStartPos, + ENH_BLOCKL); + + /* Calculate the rescaling factor for the correlation in order to + put the correlation in a int16_t vector instead */ + maxtemp = WebRtcSpl_MaxAbsValueW32(corrVecTemp, corrdim); + + scalefact = WebRtcSpl_GetSizeInBits(maxtemp) - 15; + + if (scalefact > 0) { + for (i = 0; i < corrdim; i++) { + corrVec[i] = (int16_t)(corrVecTemp[i] >> scalefact); + } + } else { + for (i = 0; i < corrdim; i++) { + corrVec[i] = (int16_t)corrVecTemp[i]; + } + } + /* In order to guarantee that all values are initialized */ + for (i = corrdim; i < ENH_CORRDIM; i++) { + corrVec[i] = 0; + } + + /* Upsample the correlation */ + WebRtcIlbcfix_EnhUpsample(corrVecUps, corrVec); + + /* Find maximum */ + tloc = WebRtcSpl_MaxIndexW32(corrVecUps, ENH_UPS0 * corrdim); + + /* make vector can be upsampled without ever running outside + bounds */ + *updStartPos = searchSegStartPos * 4 + tloc + 4; + + tloc2 = (tloc + 3) >> 2; + + /* initialize the vector to be filtered, stuff with zeros + when data is outside idata buffer */ + if (ENH_FL0 > (searchSegStartPos + tloc2)) { + const size_t st = ENH_FL0 - searchSegStartPos - tloc2; + WebRtcSpl_MemSetW16(vect, 0, st); + WEBRTC_SPL_MEMCPY_W16(&vect[st], idata, ENH_VECTL - st); + } else { + const size_t st = searchSegStartPos + tloc2 - ENH_FL0; + if ((st + ENH_VECTL) > idatal) { + const size_t en = st + ENH_VECTL - idatal; + WEBRTC_SPL_MEMCPY_W16(vect, &idata[st], ENH_VECTL - en); + WebRtcSpl_MemSetW16(&vect[ENH_VECTL - en], 0, en); + } else { + WEBRTC_SPL_MEMCPY_W16(vect, &idata[st], ENH_VECTL); + } + } + + /* compute the segment (this is actually a convolution) */ + filtStatePtr = filt + 6; + polyPtr = (int16_t*)WebRtcIlbcfix_kEnhPolyPhaser[tloc2 * ENH_UPS0 - tloc]; + for (i = 0; i < 7; i++) { + *filtStatePtr-- = *polyPtr++; + } + + WebRtcSpl_FilterMAFastQ12(&vect[6], vect, filt, ENH_FLO_MULT2_PLUS1, + ENH_BLOCKL); + + /* Add the contribution from this vector (scaled with gain) to the total + surround vector */ + WebRtcSpl_AddAffineVectorToVector(surround, vect, gain, 32768, 16, + ENH_BLOCKL); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.h new file mode 100644 index 0000000000..564c9d96e6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Refiner.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_REFINER_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_REFINER_H_ + +#include +#include + +/*----------------------------------------------------------------* + * find segment starting near idata+estSegPos that has highest + * correlation with idata+centerStartPos through + * idata+centerStartPos+ENH_BLOCKL-1 segment is found at a + * resolution of ENH_UPSO times the original of the original + * sampling rate + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Refiner( + size_t* updStartPos, /* (o) updated start point (Q-2) */ + int16_t* idata, /* (i) original data buffer */ + size_t idatal, /* (i) dimension of idata */ + size_t centerStartPos, /* (i) beginning center segment */ + size_t estSegPos, /* (i) estimated beginning other segment (Q-2) */ + int16_t* surround, /* (i/o) The contribution from this sequence + summed with earlier contributions */ + int16_t gain /* (i) Gain to use for this sequence */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c new file mode 100644 index 0000000000..7343530a5e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleInterpolateLsf.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h" + +#include "modules/audio_coding/codecs/ilbc/bw_expand.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.h" + +/*----------------------------------------------------------------* + * lsf interpolator (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleInterpolateLsf( + int16_t *syntdenum, /* (o) the synthesis filter denominator + resulting from the quantized + interpolated lsf Q12 */ + int16_t *weightdenum, /* (o) the weighting filter denominator + resulting from the unquantized + interpolated lsf Q12 */ + int16_t *lsf, /* (i) the unquantized lsf coefficients Q13 */ + int16_t *lsfdeq, /* (i) the dequantized lsf coefficients Q13 */ + int16_t *lsfold, /* (i) the unquantized lsf coefficients of + the previous signal frame Q13 */ + int16_t *lsfdeqold, /* (i) the dequantized lsf coefficients of the + previous signal frame Q13 */ + int16_t length, /* (i) should equate FILTERORDER */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ) { + size_t i; + int pos, lp_length; + + int16_t *lsf2, *lsfdeq2; + /* Stack based */ + int16_t lp[LPC_FILTERORDER + 1]; + + lsf2 = lsf + length; + lsfdeq2 = lsfdeq + length; + lp_length = length + 1; + + if (iLBCenc_inst->mode==30) { + /* subframe 1: Interpolation between old and first set of + lsf coefficients */ + + /* Calculate Analysis/Syntehsis filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfdeqold, lsfdeq, + WebRtcIlbcfix_kLsfWeight30ms[0], + length); + WEBRTC_SPL_MEMCPY_W16(syntdenum, lp, lp_length); + + /* Calculate Weighting filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfold, lsf, + WebRtcIlbcfix_kLsfWeight30ms[0], + length); + WebRtcIlbcfix_BwExpand(weightdenum, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpWeightDenum, + (int16_t)lp_length); + + /* subframe 2 to 6: Interpolation between first and second + set of lsf coefficients */ + + pos = lp_length; + for (i = 1; i < iLBCenc_inst->nsub; i++) { + + /* Calculate Analysis/Syntehsis filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfdeq, lsfdeq2, + WebRtcIlbcfix_kLsfWeight30ms[i], + length); + WEBRTC_SPL_MEMCPY_W16(syntdenum + pos, lp, lp_length); + + /* Calculate Weighting filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsf, lsf2, + WebRtcIlbcfix_kLsfWeight30ms[i], + length); + WebRtcIlbcfix_BwExpand(weightdenum + pos, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpWeightDenum, + (int16_t)lp_length); + + pos += lp_length; + } + + /* update memory */ + + WEBRTC_SPL_MEMCPY_W16(lsfold, lsf2, length); + WEBRTC_SPL_MEMCPY_W16(lsfdeqold, lsfdeq2, length); + + } else { /* iLBCenc_inst->mode==20 */ + pos = 0; + for (i = 0; i < iLBCenc_inst->nsub; i++) { + + /* Calculate Analysis/Syntehsis filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfdeqold, lsfdeq, + WebRtcIlbcfix_kLsfWeight20ms[i], + length); + WEBRTC_SPL_MEMCPY_W16(syntdenum + pos, lp, lp_length); + + /* Calculate Weighting filter from quantized LSF */ + WebRtcIlbcfix_LsfInterpolate2PloyEnc(lp, lsfold, lsf, + WebRtcIlbcfix_kLsfWeight20ms[i], + length); + WebRtcIlbcfix_BwExpand(weightdenum+pos, lp, + (int16_t*)WebRtcIlbcfix_kLpcChirpWeightDenum, + (int16_t)lp_length); + + pos += lp_length; + } + + /* update memory */ + + WEBRTC_SPL_MEMCPY_W16(lsfold, lsf, length); + WEBRTC_SPL_MEMCPY_W16(lsfdeqold, lsfdeq, length); + + } + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h new file mode 100644 index 0000000000..ee53e4bd08 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleInterpolateLsf.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_INTERPOLATE_LSF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_INTERPOLATE_LSF_H_ + +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * lsf interpolator (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleInterpolateLsf( + int16_t* syntdenum, /* (o) the synthesis filter denominator + resulting from the quantized + interpolated lsf Q12 */ + int16_t* weightdenum, /* (o) the weighting filter denominator + resulting from the unquantized + interpolated lsf Q12 */ + int16_t* lsf, /* (i) the unquantized lsf coefficients Q13 */ + int16_t* lsfdeq, /* (i) the dequantized lsf coefficients Q13 */ + int16_t* lsfold, /* (i) the unquantized lsf coefficients of + the previous signal frame Q13 */ + int16_t* lsfdeqold, /* (i) the dequantized lsf coefficients of the + previous signal frame Q13 */ + int16_t length, /* (i) should equate FILTERORDER */ + IlbcEncoder* iLBCenc_inst + /* (i/o) the encoder state structure */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c new file mode 100644 index 0000000000..fdc4553d95 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLpcAnalysis.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h" + +#include "modules/audio_coding/codecs/ilbc/bw_expand.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/poly_to_lsf.h" +#include "modules/audio_coding/codecs/ilbc/window32_w32.h" + +/*----------------------------------------------------------------* + * lpc analysis (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLpcAnalysis( + int16_t *lsf, /* (o) lsf coefficients */ + int16_t *data, /* (i) new block of speech */ + IlbcEncoder *iLBCenc_inst + /* (i/o) the encoder state structure */ + ) { + int k; + int scale; + size_t is; + int16_t stability; + /* Stack based */ + int16_t A[LPC_FILTERORDER + 1]; + int32_t R[LPC_FILTERORDER + 1]; + int16_t windowedData[BLOCKL_MAX]; + int16_t rc[LPC_FILTERORDER]; + + is=LPC_LOOKBACK+BLOCKL_MAX-iLBCenc_inst->blockl; + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->lpc_buffer+is,data,iLBCenc_inst->blockl); + + /* No lookahead, last window is asymmetric */ + + for (k = 0; k < iLBCenc_inst->lpc_n; k++) { + + is = LPC_LOOKBACK; + + if (k < (iLBCenc_inst->lpc_n - 1)) { + + /* Hanning table WebRtcIlbcfix_kLpcWin[] is in Q15-domain so the output is right-shifted 15 */ + WebRtcSpl_ElementwiseVectorMult(windowedData, iLBCenc_inst->lpc_buffer, WebRtcIlbcfix_kLpcWin, BLOCKL_MAX, 15); + } else { + + /* Hanning table WebRtcIlbcfix_kLpcAsymWin[] is in Q15-domain so the output is right-shifted 15 */ + WebRtcSpl_ElementwiseVectorMult(windowedData, iLBCenc_inst->lpc_buffer+is, WebRtcIlbcfix_kLpcAsymWin, BLOCKL_MAX, 15); + } + + /* Compute autocorrelation */ + WebRtcSpl_AutoCorrelation(windowedData, BLOCKL_MAX, LPC_FILTERORDER, R, &scale); + + /* Window autocorrelation vector */ + WebRtcIlbcfix_Window32W32(R, R, WebRtcIlbcfix_kLpcLagWin, LPC_FILTERORDER + 1 ); + + /* Calculate the A coefficients from the Autocorrelation using Levinson Durbin algorithm */ + stability=WebRtcSpl_LevinsonDurbin(R, A, rc, LPC_FILTERORDER); + + /* + Set the filter to {1.0, 0.0, 0.0,...} if filter from Levinson Durbin algorithm is unstable + This should basically never happen... + */ + if (stability!=1) { + A[0]=4096; + WebRtcSpl_MemSetW16(&A[1], 0, LPC_FILTERORDER); + } + + /* Bandwidth expand the filter coefficients */ + WebRtcIlbcfix_BwExpand(A, A, (int16_t*)WebRtcIlbcfix_kLpcChirpSyntDenum, LPC_FILTERORDER+1); + + /* Convert from A to LSF representation */ + WebRtcIlbcfix_Poly2Lsf(lsf + k*LPC_FILTERORDER, A); + } + + is=LPC_LOOKBACK+BLOCKL_MAX-iLBCenc_inst->blockl; + WEBRTC_SPL_MEMCPY_W16(iLBCenc_inst->lpc_buffer, + iLBCenc_inst->lpc_buffer+LPC_LOOKBACK+BLOCKL_MAX-is, is); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h new file mode 100644 index 0000000000..b5c839ba2a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLpcAnalysis.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LPC_ANALYSIS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LPC_ANALYSIS_H_ + +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * lpc analysis (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLpcAnalysis( + int16_t* lsf, /* (o) lsf coefficients */ + int16_t* data, /* (i) new block of speech */ + IlbcEncoder* iLBCenc_inst + /* (i/o) the encoder state structure */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c new file mode 100644 index 0000000000..e7494ceb59 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLsfDeQ.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * obtain dequantized lsf coefficients from quantization index + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLsfDeQ( + int16_t *lsfdeq, /* (o) dequantized lsf coefficients */ + int16_t *index, /* (i) quantization index */ + int16_t lpc_n /* (i) number of LPCs */ + ){ + int i, j, pos, cb_pos; + + /* decode first LSF */ + + pos = 0; + cb_pos = 0; + for (i = 0; i < LSF_NSPLIT; i++) { + for (j = 0; j < WebRtcIlbcfix_kLsfDimCb[i]; j++) { + lsfdeq[pos + j] = WebRtcIlbcfix_kLsfCb[cb_pos + j + index[i] * + WebRtcIlbcfix_kLsfDimCb[i]]; + } + pos += WebRtcIlbcfix_kLsfDimCb[i]; + cb_pos += WebRtcIlbcfix_kLsfSizeCb[i] * WebRtcIlbcfix_kLsfDimCb[i]; + } + + if (lpc_n>1) { + /* decode last LSF */ + pos = 0; + cb_pos = 0; + for (i = 0; i < LSF_NSPLIT; i++) { + for (j = 0; j < WebRtcIlbcfix_kLsfDimCb[i]; j++) { + lsfdeq[LPC_FILTERORDER + pos + j] = WebRtcIlbcfix_kLsfCb[ + cb_pos + index[LSF_NSPLIT + i] * WebRtcIlbcfix_kLsfDimCb[i] + j]; + } + pos += WebRtcIlbcfix_kLsfDimCb[i]; + cb_pos += WebRtcIlbcfix_kLsfSizeCb[i] * WebRtcIlbcfix_kLsfDimCb[i]; + } + } + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h new file mode 100644 index 0000000000..6d97d3df33 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLsfDeQ.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LSF_DEQUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LSF_DEQUANT_H_ + +#include + +/*----------------------------------------------------------------* + * obtain dequantized lsf coefficients from quantization index + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLsfDeQ( + int16_t* lsfdeq, /* (o) dequantized lsf coefficients */ + int16_t* index, /* (i) quantization index */ + int16_t lpc_n /* (i) number of LPCs */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c new file mode 100644 index 0000000000..1291d1442e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLsfQ.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/simple_lsf_quant.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/split_vq.h" + +/*----------------------------------------------------------------* + * lsf quantizer (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLsfQ( + int16_t *lsfdeq, /* (o) dequantized lsf coefficients + (dimension FILTERORDER) Q13 */ + int16_t *index, /* (o) quantization index */ + int16_t *lsf, /* (i) the lsf coefficient vector to be + quantized (dimension FILTERORDER) Q13 */ + int16_t lpc_n /* (i) number of lsf sets to quantize */ + ){ + + /* Quantize first LSF with memoryless split VQ */ + WebRtcIlbcfix_SplitVq( lsfdeq, index, lsf, + (int16_t*)WebRtcIlbcfix_kLsfCb, (int16_t*)WebRtcIlbcfix_kLsfDimCb, (int16_t*)WebRtcIlbcfix_kLsfSizeCb); + + if (lpc_n==2) { + /* Quantize second LSF with memoryless split VQ */ + WebRtcIlbcfix_SplitVq( lsfdeq + LPC_FILTERORDER, index + LSF_NSPLIT, + lsf + LPC_FILTERORDER, (int16_t*)WebRtcIlbcfix_kLsfCb, + (int16_t*)WebRtcIlbcfix_kLsfDimCb, (int16_t*)WebRtcIlbcfix_kLsfSizeCb); + } + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.h new file mode 100644 index 0000000000..66b553213a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SimpleLsfQ.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LSF_QUANT_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SIMPLE_LSF_QUANT_H_ + +#include + +/*----------------------------------------------------------------* + * lsf quantizer (subrutine to LPCencode) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SimpleLsfQ( + int16_t* lsfdeq, /* (o) dequantized lsf coefficients + (dimension FILTERORDER) Q13 */ + int16_t* index, /* (o) quantization index */ + int16_t* lsf, /* (i) the lsf coefficient vector to be + quantized (dimension FILTERORDER) Q13 */ + int16_t lpc_n /* (i) number of lsf sets to quantize */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.c new file mode 100644 index 0000000000..631b2f432a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.c @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Smooth.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/smooth.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/smooth_out_data.h" + +/*----------------------------------------------------------------* + * find the smoothed output data + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Smooth( + int16_t *odata, /* (o) smoothed output */ + int16_t *current, /* (i) the un enhanced residual for + this block */ + int16_t *surround /* (i) The approximation from the + surrounding sequences */ + ) { + int16_t scale, scale1, scale2; + int16_t A, B, C, denomW16; + int32_t B_W32, denom, num; + int32_t errs; + int32_t w00,w10,w11, endiff, crit; + int32_t w00prim, w10prim, w11_div_w00; + int16_t w11prim; + int16_t bitsw00, bitsw10, bitsw11; + int32_t w11w00, w10w10, w00w00; + uint32_t max1, max2, max12; + + /* compute some inner products (ensure no overflow by first calculating proper scale factor) */ + + w00 = w10 = w11 = 0; + + // Calculate a right shift that will let us sum ENH_BLOCKL pairwise products + // of values from the two sequences without overflowing an int32_t. (The +1 + // in max1 and max2 are because WebRtcSpl_MaxAbsValueW16 will return 2**15 - + // 1 if the input array contains -2**15.) + max1 = WebRtcSpl_MaxAbsValueW16(current, ENH_BLOCKL) + 1; + max2 = WebRtcSpl_MaxAbsValueW16(surround, ENH_BLOCKL) + 1; + max12 = WEBRTC_SPL_MAX(max1, max2); + scale = (64 - 31) - + WebRtcSpl_CountLeadingZeros64((max12 * max12) * (uint64_t)ENH_BLOCKL); + scale=WEBRTC_SPL_MAX(0, scale); + + w00=WebRtcSpl_DotProductWithScale(current,current,ENH_BLOCKL,scale); + w11=WebRtcSpl_DotProductWithScale(surround,surround,ENH_BLOCKL,scale); + w10=WebRtcSpl_DotProductWithScale(surround,current,ENH_BLOCKL,scale); + + if (w00<0) w00 = WEBRTC_SPL_WORD32_MAX; + if (w11<0) w11 = WEBRTC_SPL_WORD32_MAX; + + /* Rescale w00 and w11 to w00prim and w11prim, so that w00prim/w11prim + is in Q16 */ + + bitsw00 = WebRtcSpl_GetSizeInBits(w00); + bitsw11 = WebRtcSpl_GetSizeInBits(w11); + bitsw10 = WebRtcSpl_GetSizeInBits(WEBRTC_SPL_ABS_W32(w10)); + scale1 = 31 - bitsw00; + scale2 = 15 - bitsw11; + + if (scale2>(scale1-16)) { + scale2 = scale1 - 16; + } else { + scale1 = scale2 + 16; + } + + w00prim = w00 << scale1; + w11prim = (int16_t) WEBRTC_SPL_SHIFT_W32(w11, scale2); + + /* Perform C = sqrt(w11/w00) (C is in Q11 since (16+6)/2=11) */ + if (w11prim>64) { + endiff = WebRtcSpl_DivW32W16(w00prim, w11prim) << 6; + C = (int16_t)WebRtcSpl_SqrtFloor(endiff); /* C is in Q11 */ + } else { + C = 1; + } + + /* first try enhancement without power-constraint */ + + errs = WebRtcIlbcfix_Smooth_odata(odata, current, surround, C); + + + + /* if constraint violated by first try, add constraint */ + + if ( (6-scale+scale1) > 31) { + crit=0; + } else { + /* crit = 0.05 * w00 (Result in Q-6) */ + crit = WEBRTC_SPL_SHIFT_W32( + WEBRTC_SPL_MUL(ENH_A0, w00prim >> 14), + -(6-scale+scale1)); + } + + if (errs > crit) { + + if( w00 < 1) { + w00=1; + } + + /* Calculate w11*w00, w10*w10 and w00*w00 in the same Q domain */ + + scale1 = bitsw00-15; + scale2 = bitsw11-15; + + if (scale2>scale1) { + scale = scale2; + } else { + scale = scale1; + } + + w11w00 = (int16_t)WEBRTC_SPL_SHIFT_W32(w11, -scale) * + (int16_t)WEBRTC_SPL_SHIFT_W32(w00, -scale); + + w10w10 = (int16_t)WEBRTC_SPL_SHIFT_W32(w10, -scale) * + (int16_t)WEBRTC_SPL_SHIFT_W32(w10, -scale); + + w00w00 = (int16_t)WEBRTC_SPL_SHIFT_W32(w00, -scale) * + (int16_t)WEBRTC_SPL_SHIFT_W32(w00, -scale); + + /* Calculate (w11*w00-w10*w10)/(w00*w00) in Q16 */ + if (w00w00>65536) { + endiff = (w11w00-w10w10); + endiff = WEBRTC_SPL_MAX(0, endiff); + /* denom is in Q16 */ + denom = WebRtcSpl_DivW32W16(endiff, (int16_t)(w00w00 >> 16)); + } else { + denom = 65536; + } + + if( denom > 7){ /* eliminates numerical problems + for if smooth */ + + scale=WebRtcSpl_GetSizeInBits(denom)-15; + + if (scale>0) { + /* denomW16 is in Q(16+scale) */ + denomW16 = (int16_t)(denom >> scale); + + /* num in Q(34-scale) */ + num = ENH_A0_MINUS_A0A0DIV4 >> scale; + } else { + /* denomW16 is in Q16 */ + denomW16=(int16_t)denom; + + /* num in Q34 */ + num=ENH_A0_MINUS_A0A0DIV4; + } + + /* A sqrt( (ENH_A0-(ENH_A0^2)/4)*(w00*w00)/(w11*w00 + w10*w10) ) in Q9 */ + A = (int16_t)WebRtcSpl_SqrtFloor(WebRtcSpl_DivW32W16(num, denomW16)); + + /* B_W32 is in Q30 ( B = 1 - ENH_A0/2 - A * w10/w00 ) */ + scale1 = 31-bitsw10; + scale2 = 21-scale1; + w10prim = w10 == 0 ? 0 : w10 * (1 << scale1); + w00prim = WEBRTC_SPL_SHIFT_W32(w00, -scale2); + scale = bitsw00-scale2-15; + + if (scale>0) { + w10prim >>= scale; + w00prim >>= scale; + } + + if ((w00prim>0)&&(w10prim>0)) { + w11_div_w00=WebRtcSpl_DivW32W16(w10prim, (int16_t)w00prim); + + if (WebRtcSpl_GetSizeInBits(w11_div_w00)+WebRtcSpl_GetSizeInBits(A)>31) { + B_W32 = 0; + } else { + B_W32 = (int32_t)1073741824 - (int32_t)ENH_A0DIV2 - + WEBRTC_SPL_MUL(A, w11_div_w00); + } + B = (int16_t)(B_W32 >> 16); /* B in Q14. */ + } else { + /* No smoothing */ + A = 0; + B = 16384; /* 1 in Q14 */ + } + } + else{ /* essentially no difference between cycles; + smoothing not needed */ + + A = 0; + B = 16384; /* 1 in Q14 */ + } + + /* create smoothed sequence */ + + WebRtcSpl_ScaleAndAddVectors(surround, A, 9, + current, B, 14, + odata, ENH_BLOCKL); + } + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.h new file mode 100644 index 0000000000..c8752be64f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Smooth.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SMOOTH_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SMOOTH_H_ + +#include + +/*----------------------------------------------------------------* + * find the smoothed output data + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Smooth(int16_t* odata, /* (o) smoothed output */ + int16_t* current, /* (i) the un enhanced residual for + this block */ + int16_t* surround /* (i) The approximation from the + surrounding sequences */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c new file mode 100644 index 0000000000..9f952bfb93 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Smooth_odata.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/smooth_out_data.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "rtc_base/sanitizer.h" + +// An s32 + s32 -> s32 addition that's allowed to overflow. (It's still +// undefined behavior, so not a good idea; this just makes UBSan ignore the +// violation, so that our old code can continue to do what it's always been +// doing.) +static inline int32_t RTC_NO_SANITIZE("signed-integer-overflow") + OverflowingAdd_S32_S32_To_S32(int32_t a, int32_t b) { + return a + b; +} + +int32_t WebRtcIlbcfix_Smooth_odata( + int16_t *odata, + int16_t *psseq, + int16_t *surround, + int16_t C) +{ + int i; + + int16_t err; + int32_t errs; + + for(i=0;i<80;i++) { + odata[i]= (int16_t)((C * surround[i] + 1024) >> 11); + } + + errs=0; + for(i=0;i<80;i++) { + err = (psseq[i] - odata[i]) >> 3; + errs = OverflowingAdd_S32_S32_To_S32(errs, err * err); // errs in Q-6 + } + + return errs; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.h new file mode 100644 index 0000000000..318e7b04a2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Smooth_odata.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SMOOTH_OUT_DATA_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SMOOTH_OUT_DATA_H_ + +#include + +/*----------------------------------------------------------------* + * help function to WebRtcIlbcfix_Smooth() + *---------------------------------------------------------------*/ + +int32_t WebRtcIlbcfix_Smooth_odata(int16_t* odata, + int16_t* psseq, + int16_t* surround, + int16_t C); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.c new file mode 100644 index 0000000000..c3a24750f0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SortSq.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/sort_sq.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * scalar quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SortSq( + int16_t *xq, /* (o) the quantized value */ + int16_t *index, /* (o) the quantization index */ + int16_t x, /* (i) the value to quantize */ + const int16_t *cb, /* (i) the quantization codebook */ + int16_t cb_size /* (i) the size of the quantization codebook */ + ){ + int i; + + if (x <= cb[0]) { + *index = 0; + *xq = cb[0]; + } else { + i = 0; + while ((x > cb[i]) && (i < (cb_size-1))) { + i++; + } + + if (x > (((int32_t)cb[i] + cb[i - 1] + 1) >> 1)) { + *index = i; + *xq = cb[i]; + } else { + *index = i - 1; + *xq = cb[i - 1]; + } + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.h new file mode 100644 index 0000000000..02028dae93 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SortSq.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SORT_SQ_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SORT_SQ_H_ + +#include + +/*----------------------------------------------------------------* + * scalar quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SortSq( + int16_t* xq, /* (o) the quantized value */ + int16_t* index, /* (o) the quantization index */ + int16_t x, /* (i) the value to quantize */ + const int16_t* cb, /* (i) the quantization codebook */ + int16_t cb_size /* (i) the size of the quantization codebook */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.c new file mode 100644 index 0000000000..c1f04d2287 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SplitVq.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/split_vq.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/vq3.h" +#include "modules/audio_coding/codecs/ilbc/vq4.h" + +/*----------------------------------------------------------------* + * split vector quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SplitVq( + int16_t *qX, /* (o) the quantized vector in Q13 */ + int16_t *index, /* (o) a vector of indexes for all vector + codebooks in the split */ + int16_t *X, /* (i) the vector to quantize */ + int16_t *CB, /* (i) the quantizer codebook in Q13 */ + int16_t *dim, /* (i) the dimension of X and qX */ + int16_t *cbsize /* (i) the number of vectors in the codebook */ + ) { + + int16_t *qXPtr, *indexPtr, *CBPtr, *XPtr; + + /* Quantize X with the 3 vectror quantization tables */ + + qXPtr=qX; + indexPtr=index; + CBPtr=CB; + XPtr=X; + WebRtcIlbcfix_Vq3(qXPtr, indexPtr, CBPtr, XPtr, cbsize[0]); + + qXPtr+=3; + indexPtr+=1; + CBPtr+=(dim[0]*cbsize[0]); + XPtr+=3; + WebRtcIlbcfix_Vq3(qXPtr, indexPtr, CBPtr, XPtr, cbsize[1]); + + qXPtr+=3; + indexPtr+=1; + CBPtr+=(dim[1]*cbsize[1]); + XPtr+=3; + WebRtcIlbcfix_Vq4(qXPtr, indexPtr, CBPtr, XPtr, cbsize[2]); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.h new file mode 100644 index 0000000000..e4b02a2bc2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SplitVq.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SPLIT_VQ_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SPLIT_VQ_H_ + +#include + +/*----------------------------------------------------------------* + * split vector quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SplitVq( + int16_t* qX, /* (o) the quantized vector in Q13 */ + int16_t* index, /* (o) a vector of indexes for all vector + codebooks in the split */ + int16_t* X, /* (i) the vector to quantize */ + int16_t* CB, /* (i) the quantizer codebook in Q13 */ + int16_t* dim, /* (i) the dimension of X and qX */ + int16_t* cbsize /* (i) the number of vectors in the codebook */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_construct.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_construct.c new file mode 100644 index 0000000000..c58086c03b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_construct.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_StateConstruct.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/state_construct.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * decoding of the start state + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_StateConstruct( + size_t idxForMax, /* (i) 6-bit index for the quantization of + max amplitude */ + int16_t *idxVec, /* (i) vector of quantization indexes */ + int16_t *syntDenum, /* (i) synthesis filter denumerator */ + int16_t *Out_fix, /* (o) the decoded state vector */ + size_t len /* (i) length of a state vector */ + ) { + size_t k; + int16_t maxVal; + int16_t *tmp1, *tmp2, *tmp3; + /* Stack based */ + int16_t numerator[1+LPC_FILTERORDER]; + int16_t sampleValVec[2*STATE_SHORT_LEN_30MS+LPC_FILTERORDER]; + int16_t sampleMaVec[2*STATE_SHORT_LEN_30MS+LPC_FILTERORDER]; + int16_t *sampleVal = &sampleValVec[LPC_FILTERORDER]; + int16_t *sampleMa = &sampleMaVec[LPC_FILTERORDER]; + int16_t *sampleAr = &sampleValVec[LPC_FILTERORDER]; + + /* initialization of coefficients */ + + for (k=0; k> + 22); + tmp1++; + tmp2--; + } + } else if (idxForMax<59) { + for(k=0; k> + 19); + tmp1++; + tmp2--; + } + } else { + for(k=0; k> + 17); + tmp1++; + tmp2--; + } + } + + /* Set the rest of the data to zero */ + WebRtcSpl_MemSetW16(&sampleVal[len], 0, len); + + /* circular convolution with all-pass filter */ + + /* Set the state to zero */ + WebRtcSpl_MemSetW16(sampleValVec, 0, (LPC_FILTERORDER)); + + /* Run MA filter + AR filter */ + WebRtcSpl_FilterMAFastQ12( + sampleVal, sampleMa, + numerator, LPC_FILTERORDER+1, len + LPC_FILTERORDER); + WebRtcSpl_MemSetW16(&sampleMa[len + LPC_FILTERORDER], 0, (len - LPC_FILTERORDER)); + WebRtcSpl_FilterARFastQ12( + sampleMa, sampleAr, + syntDenum, LPC_FILTERORDER+1, 2 * len); + + tmp1 = &sampleAr[len-1]; + tmp2 = &sampleAr[2*len-1]; + tmp3 = Out_fix; + for(k=0;k +#include + +/*----------------------------------------------------------------* + * Generate the start state from the quantized indexes + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_StateConstruct( + size_t idxForMax, /* (i) 6-bit index for the quantization of + max amplitude */ + int16_t* idxVec, /* (i) vector of quantization indexes */ + int16_t* syntDenum, /* (i) synthesis filter denumerator */ + int16_t* Out_fix, /* (o) the decoded state vector */ + size_t len /* (i) length of a state vector */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.c new file mode 100644 index 0000000000..7227ac9d45 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_StateSearch.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/state_search.h" + +#include "modules/audio_coding/codecs/ilbc/abs_quant.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * encoding of start state + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_StateSearch( + IlbcEncoder *iLBCenc_inst, + /* (i) Encoder instance */ + iLBC_bits *iLBC_encbits,/* (i/o) Encoded bits (output idxForMax + and idxVec, input state_first) */ + int16_t *residual, /* (i) target residual vector */ + int16_t *syntDenum, /* (i) lpc synthesis filter */ + int16_t *weightDenum /* (i) weighting filter denuminator */ + ) { + size_t k, index; + int16_t maxVal; + int16_t scale, shift; + int32_t maxValsq; + int16_t scaleRes; + int16_t max; + int i; + /* Stack based */ + int16_t numerator[1+LPC_FILTERORDER]; + int16_t residualLongVec[2*STATE_SHORT_LEN_30MS+LPC_FILTERORDER]; + int16_t sampleMa[2*STATE_SHORT_LEN_30MS]; + int16_t *residualLong = &residualLongVec[LPC_FILTERORDER]; + int16_t *sampleAr = residualLong; + + /* Scale to maximum 12 bits to avoid saturation in circular convolution filter */ + max = WebRtcSpl_MaxAbsValueW16(residual, iLBCenc_inst->state_short_len); + scaleRes = WebRtcSpl_GetSizeInBits(max)-12; + scaleRes = WEBRTC_SPL_MAX(0, scaleRes); + /* Set up the filter coefficients for the circular convolution */ + for (i=0; i>scaleRes); + } + + /* Copy the residual to a temporary buffer that we can filter + * and set the remaining samples to zero. + */ + WEBRTC_SPL_MEMCPY_W16(residualLong, residual, iLBCenc_inst->state_short_len); + WebRtcSpl_MemSetW16(residualLong + iLBCenc_inst->state_short_len, 0, iLBCenc_inst->state_short_len); + + /* Run the Zero-Pole filter (Ciurcular convolution) */ + WebRtcSpl_MemSetW16(residualLongVec, 0, LPC_FILTERORDER); + WebRtcSpl_FilterMAFastQ12(residualLong, sampleMa, numerator, + LPC_FILTERORDER + 1, + iLBCenc_inst->state_short_len + LPC_FILTERORDER); + WebRtcSpl_MemSetW16(&sampleMa[iLBCenc_inst->state_short_len + LPC_FILTERORDER], 0, iLBCenc_inst->state_short_len - LPC_FILTERORDER); + + WebRtcSpl_FilterARFastQ12( + sampleMa, sampleAr, + syntDenum, LPC_FILTERORDER+1, 2 * iLBCenc_inst->state_short_len); + + for(k=0;kstate_short_len;k++){ + sampleAr[k] += sampleAr[k+iLBCenc_inst->state_short_len]; + } + + /* Find maximum absolute value in the vector */ + maxVal=WebRtcSpl_MaxAbsValueW16(sampleAr, iLBCenc_inst->state_short_len); + + /* Find the best index */ + + if ((((int32_t)maxVal)<=WebRtcIlbcfix_kChooseFrgQuant[i]) { + index=i+1; + } else { + i=63; + } + } + iLBC_encbits->idxForMax=index; + + /* Rescale the vector before quantization */ + scale=WebRtcIlbcfix_kScale[index]; + + if (index<27) { /* scale table is in Q16, fout[] is in Q(-1) and we want the result to be in Q11 */ + shift=4; + } else { /* scale table is in Q21, fout[] is in Q(-1) and we want the result to be in Q11 */ + shift=9; + } + + /* Set up vectors for AbsQuant and rescale it with the scale factor */ + WebRtcSpl_ScaleVectorWithSat(sampleAr, sampleAr, scale, + iLBCenc_inst->state_short_len, (int16_t)(shift-scaleRes)); + + /* Quantize the values in fout[] */ + WebRtcIlbcfix_AbsQuant(iLBCenc_inst, iLBC_encbits, sampleAr, weightDenum); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.h new file mode 100644 index 0000000000..6469138a0e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_StateSearch.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_STATE_SEARCH_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_STATE_SEARCH_H_ + +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * encoding of start state + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_StateSearch( + IlbcEncoder* iLBCenc_inst, + /* (i) Encoder instance */ + iLBC_bits* iLBC_encbits, /* (i/o) Encoded bits (output idxForMax + and idxVec, input state_first) */ + int16_t* residual, /* (i) target residual vector */ + int16_t* syntDenum, /* (i) lpc synthesis filter */ + int16_t* weightDenum /* (i) weighting filter denuminator */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c new file mode 100644 index 0000000000..bbafc1a2ed --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SwapBytes.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/swap_bytes.h" + +/*----------------------------------------------------------------* + * Swap bytes (to simplify operations on Little Endian machines) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SwapBytes( + const uint16_t* input, /* (i) the sequence to swap */ + size_t wordLength, /* (i) number or uint16_t to swap */ + uint16_t* output /* (o) the swapped sequence */ + ) { + size_t k; + for (k = wordLength; k > 0; k--) { + *output++ = (*input >> 8)|(*input << 8); + input++; + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.h new file mode 100644 index 0000000000..c59bf3068a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_SwapBytes.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SWAP_BYTES_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_SWAP_BYTES_H_ + +#include +#include + +/*----------------------------------------------------------------* + * Swap bytes (to simplify operations on Little Endian machines) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_SwapBytes( + const uint16_t* input, /* (i) the sequence to swap */ + size_t wordLength, /* (i) number or uint16_t to swap */ + uint16_t* output /* (o) the swapped sequence */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/empty.cc b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/empty.cc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_test.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_test.c new file mode 100644 index 0000000000..e0ca075eda --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_test.c @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + iLBC_test.c + +******************************************************************/ + +#include +#include +#include +#include "modules/audio_coding/codecs/ilbc/ilbc.h" + +/*---------------------------------------------------------------* + * Main program to test iLBC encoding and decoding + * + * Usage: + * exefile_name.exe + * + * : Input file, speech for encoder (16-bit pcm file) + * : Bit stream output from the encoder + * : Output file, decoded speech (16-bit pcm file) + * : Bit error file, optional (16-bit) + * 1 - Packet received correctly + * 0 - Packet Lost + * + *--------------------------------------------------------------*/ + +#define BLOCKL_MAX 240 +#define ILBCNOOFWORDS_MAX 25 + + +int main(int argc, char* argv[]) +{ + + FILE *ifileid,*efileid,*ofileid, *cfileid; + int16_t data[BLOCKL_MAX]; + uint8_t encoded_data[2 * ILBCNOOFWORDS_MAX]; + int16_t decoded_data[BLOCKL_MAX]; + int len_int, mode; + short pli; + int blockcount = 0; + size_t frameLen, len, len_i16s; + int16_t speechType; + IlbcEncoderInstance *Enc_Inst; + IlbcDecoderInstance *Dec_Inst; + +#ifdef __ILBC_WITH_40BITACC + /* Doublecheck that long long exists */ + if (sizeof(long)>=sizeof(long long)) { + fprintf(stderr, "40-bit simulation is not be supported on this platform\n"); + exit(0); + } +#endif + + /* get arguments and open files */ + + if ((argc!=5) && (argc!=6)) { + fprintf(stderr, + "\n*-----------------------------------------------*\n"); + fprintf(stderr, + " %s <20,30> input encoded decoded (channel)\n\n", + argv[0]); + fprintf(stderr, + " mode : Frame size for the encoding/decoding\n"); + fprintf(stderr, + " 20 - 20 ms\n"); + fprintf(stderr, + " 30 - 30 ms\n"); + fprintf(stderr, + " input : Speech for encoder (16-bit pcm file)\n"); + fprintf(stderr, + " encoded : Encoded bit stream\n"); + fprintf(stderr, + " decoded : Decoded speech (16-bit pcm file)\n"); + fprintf(stderr, + " channel : Packet loss pattern, optional (16-bit)\n"); + fprintf(stderr, + " 1 - Packet received correctly\n"); + fprintf(stderr, + " 0 - Packet Lost\n"); + fprintf(stderr, + "*-----------------------------------------------*\n\n"); + exit(1); + } + mode=atoi(argv[1]); + if (mode != 20 && mode != 30) { + fprintf(stderr,"Wrong mode %s, must be 20, or 30\n", + argv[1]); + exit(2); + } + if ( (ifileid=fopen(argv[2],"rb")) == NULL) { + fprintf(stderr,"Cannot open input file %s\n", argv[2]); + exit(2);} + if ( (efileid=fopen(argv[3],"wb")) == NULL) { + fprintf(stderr, "Cannot open encoded file file %s\n", + argv[3]); exit(1);} + if ( (ofileid=fopen(argv[4],"wb")) == NULL) { + fprintf(stderr, "Cannot open decoded file %s\n", + argv[4]); exit(1);} + if (argc==6) { + if( (cfileid=fopen(argv[5],"rb")) == NULL) { + fprintf(stderr, "Cannot open channel file %s\n", + argv[5]); + exit(1); + } + } else { + cfileid=NULL; + } + + /* print info */ + + fprintf(stderr, "\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* iLBC test program *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); + fprintf(stderr,"\nMode : %2d ms\n", mode); + fprintf(stderr,"Input file : %s\n", argv[2]); + fprintf(stderr,"Encoded file : %s\n", argv[3]); + fprintf(stderr,"Output file : %s\n", argv[4]); + if (argc==6) { + fprintf(stderr,"Channel file : %s\n", argv[5]); + } + fprintf(stderr,"\n"); + + /* Create structs */ + WebRtcIlbcfix_EncoderCreate(&Enc_Inst); + WebRtcIlbcfix_DecoderCreate(&Dec_Inst); + + + /* Initialization */ + + WebRtcIlbcfix_EncoderInit(Enc_Inst, mode); + WebRtcIlbcfix_DecoderInit(Dec_Inst, mode); + frameLen = (size_t)(mode*8); + + /* loop over input blocks */ + + while (fread(data,sizeof(int16_t),frameLen,ifileid) == frameLen) { + + blockcount++; + + /* encoding */ + + fprintf(stderr, "--- Encoding block %i --- ",blockcount); + len_int = WebRtcIlbcfix_Encode(Enc_Inst, data, frameLen, encoded_data); + if (len_int < 0) { + fprintf(stderr, "Error encoding\n"); + exit(0); + } + len = (size_t)len_int; + fprintf(stderr, "\r"); + + /* write byte file */ + + len_i16s = (len + 1) / sizeof(int16_t); + if (fwrite(encoded_data, sizeof(int16_t), len_i16s, efileid) != len_i16s) { + return -1; + } + + /* get channel data if provided */ + if (argc==6) { + if (fread(&pli, sizeof(int16_t), 1, cfileid)) { + if ((pli!=0)&&(pli!=1)) { + fprintf(stderr, "Error in channel file\n"); + exit(0); + } + if (pli==0) { + /* Packet loss -> remove info from frame */ + memset(encoded_data, 0, + sizeof(int16_t)*ILBCNOOFWORDS_MAX); + } + } else { + fprintf(stderr, "Error. Channel file too short\n"); + exit(0); + } + } else { + pli=1; + } + + /* decoding */ + + fprintf(stderr, "--- Decoding block %i --- ",blockcount); + if (pli==1) { + len_int=WebRtcIlbcfix_Decode(Dec_Inst, encoded_data, + len, decoded_data,&speechType); + if (len_int < 0) { + fprintf(stderr, "Error decoding\n"); + exit(0); + } + len = (size_t)len_int; + } else { + len=WebRtcIlbcfix_DecodePlc(Dec_Inst, decoded_data, 1); + } + fprintf(stderr, "\r"); + + /* write output file */ + + if (fwrite(decoded_data, sizeof(int16_t), len, ofileid) != len) { + return -1; + } + } + + /* close files */ + + fclose(ifileid); fclose(efileid); fclose(ofileid); + if (argc==6) { + fclose(cfileid); + } + + /* Free structs */ + WebRtcIlbcfix_EncoderFree(Enc_Inst); + WebRtcIlbcfix_DecoderFree(Dec_Inst); + + + printf("\nDone with simulation\n\n"); + + return(0); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testLib.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testLib.c new file mode 100644 index 0000000000..132f3bdb37 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testLib.c @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + +iLBC Speech Coder ANSI-C Source Code + +iLBC_test.c + +******************************************************************/ + +#include +#include +#include +#include +#include +#include "modules/audio_coding/codecs/ilbc/ilbc.h" + +//#define JUNK_DATA +#ifdef JUNK_DATA +#define SEED_FILE "randseed.txt" +#endif + + +/*----------------------------------------------------------------* +* Main program to test iLBC encoding and decoding +* +* Usage: +* exefile_name.exe +* +*---------------------------------------------------------------*/ + +int main(int argc, char* argv[]) +{ + FILE *ifileid,*efileid,*ofileid, *chfileid; + short encoded_data[55], data[240], speechType; + int len_int, mode; + short pli; + size_t len, readlen; + int blockcount = 0; + + IlbcEncoderInstance *Enc_Inst; + IlbcDecoderInstance *Dec_Inst; +#ifdef JUNK_DATA + size_t i; + FILE *seedfile; + unsigned int random_seed = (unsigned int) time(NULL);//1196764538 +#endif + + /* Create structs */ + WebRtcIlbcfix_EncoderCreate(&Enc_Inst); + WebRtcIlbcfix_DecoderCreate(&Dec_Inst); + + /* get arguments and open files */ + + if (argc != 6 ) { + fprintf(stderr, "%s mode inputfile bytefile outputfile channelfile\n", + argv[0]); + fprintf(stderr, "Example:\n"); + fprintf(stderr, "%s <30,20> in.pcm byte.dat out.pcm T30.0.dat\n", argv[0]); + exit(1); + } + mode=atoi(argv[1]); + if (mode != 20 && mode != 30) { + fprintf(stderr,"Wrong mode %s, must be 20, or 30\n", argv[1]); + exit(2); + } + if ( (ifileid=fopen(argv[2],"rb")) == NULL) { + fprintf(stderr,"Cannot open input file %s\n", argv[2]); + exit(2);} + if ( (efileid=fopen(argv[3],"wb")) == NULL) { + fprintf(stderr, "Cannot open channelfile file %s\n", + argv[3]); exit(3);} + if( (ofileid=fopen(argv[4],"wb")) == NULL) { + fprintf(stderr, "Cannot open output file %s\n", + argv[4]); exit(3);} + if ( (chfileid=fopen(argv[5],"rb")) == NULL) { + fprintf(stderr,"Cannot open channel file file %s\n", argv[5]); + exit(2); + } + /* print info */ + fprintf(stderr, "\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* iLBCtest *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); +#ifdef SPLIT_10MS + fprintf(stderr,"\n10ms split with raw mode: %2d ms\n", mode); +#else + fprintf(stderr,"\nMode : %2d ms\n", mode); +#endif + fprintf(stderr,"\nInput file : %s\n", argv[2]); + fprintf(stderr,"Coded file : %s\n", argv[3]); + fprintf(stderr,"Output file : %s\n\n", argv[4]); + fprintf(stderr,"Channel file : %s\n\n", argv[5]); + +#ifdef JUNK_DATA + srand(random_seed); + + if ( (seedfile = fopen(SEED_FILE, "a+t") ) == NULL ) { + fprintf(stderr, "Error: Could not open file %s\n", SEED_FILE); + } + else { + fprintf(seedfile, "%u\n", random_seed); + fclose(seedfile); + } +#endif + + /* Initialization */ + WebRtcIlbcfix_EncoderInit(Enc_Inst, mode); + WebRtcIlbcfix_DecoderInit(Dec_Inst, mode); + + /* loop over input blocks */ +#ifdef SPLIT_10MS + readlen = 80; +#else + readlen = (size_t)(mode << 3); +#endif + while(fread(data, sizeof(short), readlen, ifileid) == readlen) { + blockcount++; + + /* encoding */ + fprintf(stderr, "--- Encoding block %i --- ",blockcount); + len_int=WebRtcIlbcfix_Encode(Enc_Inst, data, readlen, encoded_data); + if (len_int < 0) { + fprintf(stderr, "Error encoding\n"); + exit(0); + } + len = (size_t)len_int; + fprintf(stderr, "\r"); + +#ifdef JUNK_DATA + for ( i = 0; i < len; i++) { + encoded_data[i] = (short) (encoded_data[i] + (short) rand()); + } +#endif + /* write byte file */ + if(len != 0){ //len may be 0 in 10ms split case + fwrite(encoded_data,1,len,efileid); + + /* get channel data if provided */ + if (argc==6) { + if (fread(&pli, sizeof(int16_t), 1, chfileid)) { + if ((pli!=0)&&(pli!=1)) { + fprintf(stderr, "Error in channel file\n"); + exit(0); + } + if (pli==0) { + /* Packet loss -> remove info from frame */ + memset(encoded_data, 0, sizeof(int16_t)*25); + } + } else { + fprintf(stderr, "Error. Channel file too short\n"); + exit(0); + } + } else { + pli=1; + } + + /* decoding */ + fprintf(stderr, "--- Decoding block %i --- ",blockcount); + if (pli==1) { + len_int = WebRtcIlbcfix_Decode(Dec_Inst, encoded_data, len, data, + &speechType); + if (len_int < 0) { + fprintf(stderr, "Error decoding\n"); + exit(0); + } + len = (size_t)len_int; + } else { + len=WebRtcIlbcfix_DecodePlc(Dec_Inst, data, 1); + } + fprintf(stderr, "\r"); + + /* write output file */ + fwrite(data,sizeof(short),len,ofileid); + } + } + +#ifdef JUNK_DATA + if ( (seedfile = fopen(SEED_FILE, "a+t") ) == NULL ) { + fprintf(stderr, "Error: Could not open file %s\n", SEED_FILE); + } + else { + fprintf(seedfile, "ok\n\n"); + fclose(seedfile); + } +#endif + + /* free structs */ + WebRtcIlbcfix_EncoderFree(Enc_Inst); + WebRtcIlbcfix_DecoderFree(Dec_Inst); + + /* close files */ + fclose(ifileid); + fclose(efileid); + fclose(ofileid); + + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testprogram.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testprogram.c new file mode 100644 index 0000000000..a62a42edf6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/test/iLBC_testprogram.c @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + iLBC_test.c + +******************************************************************/ + +#include +#include +#include +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" +#include "modules/audio_coding/codecs/ilbc/nit_encode.h" +#include "modules/audio_coding/codecs/ilbc/encode.h" +#include "modules/audio_coding/codecs/ilbc/init_decode.h" +#include "modules/audio_coding/codecs/ilbc/decode.h" +#include "modules/audio_coding/codecs/ilbc/constants.h" +#include "modules/audio_coding/codecs/ilbc/ilbc.h" + +#define ILBCNOOFWORDS_MAX (NO_OF_BYTES_30MS)/2 + +/* Runtime statistics */ +#include +/* #define CLOCKS_PER_SEC 1000 */ + +/*----------------------------------------------------------------* + * Encoder interface function + *---------------------------------------------------------------*/ + +short encode( /* (o) Number of bytes encoded */ + IlbcEncoder *iLBCenc_inst, /* (i/o) Encoder instance */ + int16_t *encoded_data, /* (o) The encoded bytes */ + int16_t *data /* (i) The signal block to encode */ + ){ + + /* do the actual encoding */ + WebRtcIlbcfix_Encode((uint16_t *)encoded_data, data, iLBCenc_inst); + + return (iLBCenc_inst->no_of_bytes); +} + +/*----------------------------------------------------------------* + * Decoder interface function + *---------------------------------------------------------------*/ + +short decode( /* (o) Number of decoded samples */ + IlbcDecoder *iLBCdec_inst, /* (i/o) Decoder instance */ + short *decoded_data, /* (o) Decoded signal block */ + short *encoded_data, /* (i) Encoded bytes */ + short mode /* (i) 0=PL, 1=Normal */ + ){ + + /* check if mode is valid */ + + if (mode<0 || mode>1) { + printf("\nERROR - Wrong mode - 0, 1 allowed\n"); exit(3);} + + /* do actual decoding of block */ + + WebRtcIlbcfix_Decode(decoded_data, (uint16_t *)encoded_data, + iLBCdec_inst, mode); + + return (iLBCdec_inst->blockl); +} + +/*----------------------------------------------------------------* + * Main program to test iLBC encoding and decoding + * + * Usage: + * exefile_name.exe + * + *---------------------------------------------------------------*/ + +#define MAXFRAMES 10000 +#define MAXFILELEN (BLOCKL_MAX*MAXFRAMES) + +int main(int argc, char* argv[]) +{ + + /* Runtime statistics */ + + float starttime1, starttime2; + float runtime1, runtime2; + float outtime; + + FILE *ifileid,*efileid,*ofileid, *chfileid; + short *inputdata, *encodeddata, *decodeddata; + short *channeldata; + int blockcount = 0, noOfBlocks=0, i, noOfLostBlocks=0; + short mode; + IlbcEncoder Enc_Inst; + IlbcDecoder Dec_Inst; + + short frameLen; + short count; +#ifdef SPLIT_10MS + short size; +#endif + + inputdata=(short*) malloc(MAXFILELEN*sizeof(short)); + if (inputdata==NULL) { + fprintf(stderr,"Could not allocate memory for vector\n"); + exit(0); + } + encodeddata=(short*) malloc(ILBCNOOFWORDS_MAX*MAXFRAMES*sizeof(short)); + if (encodeddata==NULL) { + fprintf(stderr,"Could not allocate memory for vector\n"); + free(inputdata); + exit(0); + } + decodeddata=(short*) malloc(MAXFILELEN*sizeof(short)); + if (decodeddata==NULL) { + fprintf(stderr,"Could not allocate memory for vector\n"); + free(inputdata); + free(encodeddata); + exit(0); + } + channeldata=(short*) malloc(MAXFRAMES*sizeof(short)); + if (channeldata==NULL) { + fprintf(stderr,"Could not allocate memory for vector\n"); + free(inputdata); + free(encodeddata); + free(decodeddata); + exit(0); + } + + /* get arguments and open files */ + + if (argc != 6 ) { + fprintf(stderr, "%s mode inputfile bytefile outputfile channelfile\n", + argv[0]); + fprintf(stderr, "Example:\n"); + fprintf(stderr, "%s <30,20> in.pcm byte.dat out.pcm T30.0.dat\n", argv[0]); + exit(1); + } + mode=atoi(argv[1]); + if (mode != 20 && mode != 30) { + fprintf(stderr,"Wrong mode %s, must be 20, or 30\n", argv[1]); + exit(2); + } + if ( (ifileid=fopen(argv[2],"rb")) == NULL) { + fprintf(stderr,"Cannot open input file %s\n", argv[2]); + exit(2);} + if ( (efileid=fopen(argv[3],"wb")) == NULL) { + fprintf(stderr, "Cannot open channelfile file %s\n", + argv[3]); exit(3);} + if( (ofileid=fopen(argv[4],"wb")) == NULL) { + fprintf(stderr, "Cannot open output file %s\n", + argv[4]); exit(3);} + if ( (chfileid=fopen(argv[5],"rb")) == NULL) { + fprintf(stderr,"Cannot open channel file file %s\n", argv[5]); + exit(2);} + + + /* print info */ +#ifndef PRINT_MIPS + fprintf(stderr, "\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* iLBCtest *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "* *\n"); + fprintf(stderr, + "*---------------------------------------------------*\n"); +#ifdef SPLIT_10MS + fprintf(stderr,"\n10ms split with raw mode: %2d ms\n", mode); +#else + fprintf(stderr,"\nMode : %2d ms\n", mode); +#endif + fprintf(stderr,"\nInput file : %s\n", argv[2]); + fprintf(stderr,"Coded file : %s\n", argv[3]); + fprintf(stderr,"Output file : %s\n\n", argv[4]); + fprintf(stderr,"Channel file : %s\n\n", argv[5]); +#endif + + /* Initialization */ + + WebRtcIlbcfix_EncoderInit(&Enc_Inst, mode); + WebRtcIlbcfix_DecoderInit(&Dec_Inst, mode, 1); + + /* extract the input file and channel file */ + +#ifdef SPLIT_10MS + frameLen = (mode==20)? 80:160; + fread(Enc_Inst.past_samples, sizeof(short), frameLen, ifileid); + Enc_Inst.section = 0; + + while( fread(&inputdata[noOfBlocks*80], sizeof(short), + 80, ifileid) == 80 ) { + noOfBlocks++; + } + + noOfBlocks += frameLen/80; + frameLen = 80; +#else + frameLen = Enc_Inst.blockl; + + while( fread(&inputdata[noOfBlocks*Enc_Inst.blockl],sizeof(short), + Enc_Inst.blockl,ifileid)==(uint16_t)Enc_Inst.blockl){ + noOfBlocks++; + } +#endif + + + while ((fread(&channeldata[blockcount],sizeof(short), 1,chfileid)==1) + && ( blockcount < noOfBlocks/(Enc_Inst.blockl/frameLen) )) { + blockcount++; + } + + if ( blockcount < noOfBlocks/(Enc_Inst.blockl/frameLen) ) { + fprintf(stderr,"Channel file %s is too short\n", argv[4]); + free(inputdata); + free(encodeddata); + free(decodeddata); + free(channeldata); + exit(0); + } + + count=0; + + /* Runtime statistics */ + + starttime1 = clock()/(float)CLOCKS_PER_SEC; + + /* Encoding loop */ +#ifdef PRINT_MIPS + printf("-1 -1\n"); +#endif + +#ifdef SPLIT_10MS + /* "Enc_Inst.section != 0" is to make sure we run through full + lengths of all vectors for 10ms split mode. + */ + // while( (count < noOfBlocks) || (Enc_Inst.section != 0) ) { + while( count < blockcount * (Enc_Inst.blockl/frameLen) ) { + + encode(&Enc_Inst, &encodeddata[Enc_Inst.no_of_words * + (count/(Enc_Inst.nsub/2))], + &inputdata[frameLen * count] ); +#else + while (count < noOfBlocks) { + encode( &Enc_Inst, &encodeddata[Enc_Inst.no_of_words * count], + &inputdata[frameLen * count] ); +#endif + +#ifdef PRINT_MIPS + printf("-1 -1\n"); +#endif + + count++; + } + + count=0; + + /* Runtime statistics */ + + starttime2=clock()/(float)CLOCKS_PER_SEC; + runtime1 = (float)(starttime2-starttime1); + + /* Decoding loop */ + + while (count < blockcount) { + if (channeldata[count]==1) { + /* Normal decoding */ + decode(&Dec_Inst, &decodeddata[count * Dec_Inst.blockl], + &encodeddata[Dec_Inst.no_of_words * count], 1); + } else if (channeldata[count]==0) { + /* PLC */ + short emptydata[ILBCNOOFWORDS_MAX]; + memset(emptydata, 0, Dec_Inst.no_of_words*sizeof(short)); + decode(&Dec_Inst, &decodeddata[count*Dec_Inst.blockl], + emptydata, 0); + noOfLostBlocks++; + } else { + printf("Error in channel file (values have to be either 1 or 0)\n"); + exit(0); + } +#ifdef PRINT_MIPS + printf("-1 -1\n"); +#endif + + count++; + } + + /* Runtime statistics */ + + runtime2 = (float)(clock()/(float)CLOCKS_PER_SEC-starttime2); + + outtime = (float)((float)blockcount* + (float)mode/1000.0); + +#ifndef PRINT_MIPS + printf("\nLength of speech file: %.1f s\n", outtime); + printf("Lost frames : %.1f%%\n\n", 100*(float)noOfLostBlocks/(float)blockcount); + + printf("Time to run iLBC_encode+iLBC_decode:"); + printf(" %.1f s (%.1f%% of realtime)\n", runtime1+runtime2, + (100*(runtime1+runtime2)/outtime)); + + printf("Time in iLBC_encode :"); + printf(" %.1f s (%.1f%% of total runtime)\n", + runtime1, 100.0*runtime1/(runtime1+runtime2)); + + printf("Time in iLBC_decode :"); + printf(" %.1f s (%.1f%% of total runtime)\n\n", + runtime2, 100.0*runtime2/(runtime1+runtime2)); +#endif + + /* Write data to files */ + for (i=0; ilsf[0] = (*bitstreamPtr)>>10; /* Bit 0..5 */ + enc_bits->lsf[1] = ((*bitstreamPtr)>>3)&0x7F; /* Bit 6..12 */ + enc_bits->lsf[2] = ((*bitstreamPtr)&0x7)<<4; /* Bit 13..15 */ + bitstreamPtr++; + /* Second int16_t */ + enc_bits->lsf[2] |= ((*bitstreamPtr)>>12)&0xF; /* Bit 0..3 */ + + if (mode==20) { + enc_bits->startIdx = ((*bitstreamPtr)>>10)&0x3; /* Bit 4..5 */ + enc_bits->state_first = ((*bitstreamPtr)>>9)&0x1; /* Bit 6 */ + enc_bits->idxForMax = ((*bitstreamPtr)>>3)&0x3F; /* Bit 7..12 */ + enc_bits->cb_index[0] = ((*bitstreamPtr)&0x7)<<4; /* Bit 13..15 */ + bitstreamPtr++; + /* Third int16_t */ + enc_bits->cb_index[0] |= ((*bitstreamPtr)>>12)&0xE; /* Bit 0..2 */ + enc_bits->gain_index[0] = ((*bitstreamPtr)>>8)&0x18; /* Bit 3..4 */ + enc_bits->gain_index[1] = ((*bitstreamPtr)>>7)&0x8; /* Bit 5 */ + enc_bits->cb_index[3] = ((*bitstreamPtr)>>2)&0xFE; /* Bit 6..12 */ + enc_bits->gain_index[3] = ((*bitstreamPtr)<<2)&0x10; /* Bit 13 */ + enc_bits->gain_index[4] = ((*bitstreamPtr)<<2)&0x8; /* Bit 14 */ + enc_bits->gain_index[6] = ((*bitstreamPtr)<<4)&0x10; /* Bit 15 */ + } else { /* mode==30 */ + enc_bits->lsf[3] = ((*bitstreamPtr)>>6)&0x3F; /* Bit 4..9 */ + enc_bits->lsf[4] = ((*bitstreamPtr)<<1)&0x7E; /* Bit 10..15 */ + bitstreamPtr++; + /* Third int16_t */ + enc_bits->lsf[4] |= ((*bitstreamPtr)>>15)&0x1; /* Bit 0 */ + enc_bits->lsf[5] = ((*bitstreamPtr)>>8)&0x7F; /* Bit 1..7 */ + enc_bits->startIdx = ((*bitstreamPtr)>>5)&0x7; /* Bit 8..10 */ + enc_bits->state_first = ((*bitstreamPtr)>>4)&0x1; /* Bit 11 */ + enc_bits->idxForMax = ((*bitstreamPtr)<<2)&0x3C; /* Bit 12..15 */ + bitstreamPtr++; + /* 4:th int16_t */ + enc_bits->idxForMax |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->cb_index[0] = ((*bitstreamPtr)>>7)&0x78; /* Bit 2..5 */ + enc_bits->gain_index[0] = ((*bitstreamPtr)>>5)&0x10; /* Bit 6 */ + enc_bits->gain_index[1] = ((*bitstreamPtr)>>5)&0x8; /* Bit 7 */ + enc_bits->cb_index[3] = ((*bitstreamPtr))&0xFC; /* Bit 8..13 */ + enc_bits->gain_index[3] = ((*bitstreamPtr)<<3)&0x10; /* Bit 14 */ + enc_bits->gain_index[4] = ((*bitstreamPtr)<<3)&0x8; /* Bit 15 */ + } + /* Class 2 bits of ULP */ + /* 4:th to 6:th int16_t for 20 ms case + 5:th to 7:th int16_t for 30 ms case */ + bitstreamPtr++; + tmpPtr=enc_bits->idxVec; + for (k=0; k<3; k++) { + for (i=15; i>=0; i--) { + (*tmpPtr) = (((*bitstreamPtr)>>i)<<2)&0x4; + /* Bit 15-i */ + tmpPtr++; + } + bitstreamPtr++; + } + + if (mode==20) { + /* 7:th int16_t */ + for (i=15; i>6; i--) { + (*tmpPtr) = (((*bitstreamPtr)>>i)<<2)&0x4; + /* Bit 15-i */ + tmpPtr++; + } + enc_bits->gain_index[1] |= ((*bitstreamPtr)>>4)&0x4; /* Bit 9 */ + enc_bits->gain_index[3] |= ((*bitstreamPtr)>>2)&0xC; /* Bit 10..11 */ + enc_bits->gain_index[4] |= ((*bitstreamPtr)>>1)&0x4; /* Bit 12 */ + enc_bits->gain_index[6] |= ((*bitstreamPtr)<<1)&0x8; /* Bit 13 */ + enc_bits->gain_index[7] = ((*bitstreamPtr)<<2)&0xC; /* Bit 14..15 */ + + } else { /* mode==30 */ + /* 8:th int16_t */ + for (i=15; i>5; i--) { + (*tmpPtr) = (((*bitstreamPtr)>>i)<<2)&0x4; + /* Bit 15-i */ + tmpPtr++; + } + enc_bits->cb_index[0] |= ((*bitstreamPtr)>>3)&0x6; /* Bit 10..11 */ + enc_bits->gain_index[0] |= ((*bitstreamPtr))&0x8; /* Bit 12 */ + enc_bits->gain_index[1] |= ((*bitstreamPtr))&0x4; /* Bit 13 */ + enc_bits->cb_index[3] |= ((*bitstreamPtr))&0x2; /* Bit 14 */ + enc_bits->cb_index[6] = ((*bitstreamPtr)<<7)&0x80; /* Bit 15 */ + bitstreamPtr++; + /* 9:th int16_t */ + enc_bits->cb_index[6] |= ((*bitstreamPtr)>>9)&0x7E; /* Bit 0..5 */ + enc_bits->cb_index[9] = ((*bitstreamPtr)>>2)&0xFE; /* Bit 6..12 */ + enc_bits->cb_index[12] = ((*bitstreamPtr)<<5)&0xE0; /* Bit 13..15 */ + bitstreamPtr++; + /* 10:th int16_t */ + enc_bits->cb_index[12] |= ((*bitstreamPtr)>>11)&0x1E;/* Bit 0..3 */ + enc_bits->gain_index[3] |= ((*bitstreamPtr)>>8)&0xC; /* Bit 4..5 */ + enc_bits->gain_index[4] |= ((*bitstreamPtr)>>7)&0x6; /* Bit 6..7 */ + enc_bits->gain_index[6] = ((*bitstreamPtr)>>3)&0x18; /* Bit 8..9 */ + enc_bits->gain_index[7] = ((*bitstreamPtr)>>2)&0xC; /* Bit 10..11 */ + enc_bits->gain_index[9] = ((*bitstreamPtr)<<1)&0x10; /* Bit 12 */ + enc_bits->gain_index[10] = ((*bitstreamPtr)<<1)&0x8; /* Bit 13 */ + enc_bits->gain_index[12] = ((*bitstreamPtr)<<3)&0x10; /* Bit 14 */ + enc_bits->gain_index[13] = ((*bitstreamPtr)<<3)&0x8; /* Bit 15 */ + } + bitstreamPtr++; + /* Class 3 bits of ULP */ + /* 8:th to 14:th int16_t for 20 ms case + 11:th to 17:th int16_t for 30 ms case */ + tmpPtr=enc_bits->idxVec; + for (k=0; k<7; k++) { + for (i=14; i>=0; i-=2) { + (*tmpPtr) |= ((*bitstreamPtr)>>i)&0x3; /* Bit 15-i..14-i*/ + tmpPtr++; + } + bitstreamPtr++; + } + + if (mode==20) { + /* 15:th int16_t */ + enc_bits->idxVec[56] |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->cb_index[0] |= ((*bitstreamPtr)>>13)&0x1; /* Bit 2 */ + enc_bits->cb_index[1] = ((*bitstreamPtr)>>6)&0x7F; /* Bit 3..9 */ + enc_bits->cb_index[2] = ((*bitstreamPtr)<<1)&0x7E; /* Bit 10..15 */ + bitstreamPtr++; + /* 16:th int16_t */ + enc_bits->cb_index[2] |= ((*bitstreamPtr)>>15)&0x1; /* Bit 0 */ + enc_bits->gain_index[0] |= ((*bitstreamPtr)>>12)&0x7; /* Bit 1..3 */ + enc_bits->gain_index[1] |= ((*bitstreamPtr)>>10)&0x3; /* Bit 4..5 */ + enc_bits->gain_index[2] = ((*bitstreamPtr)>>7)&0x7; /* Bit 6..8 */ + enc_bits->cb_index[3] |= ((*bitstreamPtr)>>6)&0x1; /* Bit 9 */ + enc_bits->cb_index[4] = ((*bitstreamPtr)<<1)&0x7E; /* Bit 10..15 */ + bitstreamPtr++; + /* 17:th int16_t */ + enc_bits->cb_index[4] |= ((*bitstreamPtr)>>15)&0x1; /* Bit 0 */ + enc_bits->cb_index[5] = ((*bitstreamPtr)>>8)&0x7F; /* Bit 1..7 */ + enc_bits->cb_index[6] = ((*bitstreamPtr))&0xFF; /* Bit 8..15 */ + bitstreamPtr++; + /* 18:th int16_t */ + enc_bits->cb_index[7] = (*bitstreamPtr)>>8; /* Bit 0..7 */ + enc_bits->cb_index[8] = (*bitstreamPtr)&0xFF; /* Bit 8..15 */ + bitstreamPtr++; + /* 19:th int16_t */ + enc_bits->gain_index[3] |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->gain_index[4] |= ((*bitstreamPtr)>>12)&0x3; /* Bit 2..3 */ + enc_bits->gain_index[5] = ((*bitstreamPtr)>>9)&0x7; /* Bit 4..6 */ + enc_bits->gain_index[6] |= ((*bitstreamPtr)>>6)&0x7; /* Bit 7..9 */ + enc_bits->gain_index[7] |= ((*bitstreamPtr)>>4)&0x3; /* Bit 10..11 */ + enc_bits->gain_index[8] = ((*bitstreamPtr)>>1)&0x7; /* Bit 12..14 */ + } else { /* mode==30 */ + /* 18:th int16_t */ + enc_bits->idxVec[56] |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->idxVec[57] |= ((*bitstreamPtr)>>12)&0x3; /* Bit 2..3 */ + enc_bits->cb_index[0] |= ((*bitstreamPtr)>>11)&1; /* Bit 4 */ + enc_bits->cb_index[1] = ((*bitstreamPtr)>>4)&0x7F; /* Bit 5..11 */ + enc_bits->cb_index[2] = ((*bitstreamPtr)<<3)&0x78; /* Bit 12..15 */ + bitstreamPtr++; + /* 19:th int16_t */ + enc_bits->cb_index[2] |= ((*bitstreamPtr)>>13)&0x7; /* Bit 0..2 */ + enc_bits->gain_index[0] |= ((*bitstreamPtr)>>10)&0x7; /* Bit 3..5 */ + enc_bits->gain_index[1] |= ((*bitstreamPtr)>>8)&0x3; /* Bit 6..7 */ + enc_bits->gain_index[2] = ((*bitstreamPtr)>>5)&0x7; /* Bit 8..10 */ + enc_bits->cb_index[3] |= ((*bitstreamPtr)>>4)&0x1; /* Bit 11 */ + enc_bits->cb_index[4] = ((*bitstreamPtr)<<3)&0x78; /* Bit 12..15 */ + bitstreamPtr++; + /* 20:th int16_t */ + enc_bits->cb_index[4] |= ((*bitstreamPtr)>>13)&0x7; /* Bit 0..2 */ + enc_bits->cb_index[5] = ((*bitstreamPtr)>>6)&0x7F; /* Bit 3..9 */ + enc_bits->cb_index[6] |= ((*bitstreamPtr)>>5)&0x1; /* Bit 10 */ + enc_bits->cb_index[7] = ((*bitstreamPtr)<<3)&0xF8; /* Bit 11..15 */ + bitstreamPtr++; + /* 21:st int16_t */ + enc_bits->cb_index[7] |= ((*bitstreamPtr)>>13)&0x7; /* Bit 0..2 */ + enc_bits->cb_index[8] = ((*bitstreamPtr)>>5)&0xFF; /* Bit 3..10 */ + enc_bits->cb_index[9] |= ((*bitstreamPtr)>>4)&0x1; /* Bit 11 */ + enc_bits->cb_index[10] = ((*bitstreamPtr)<<4)&0xF0; /* Bit 12..15 */ + bitstreamPtr++; + /* 22:nd int16_t */ + enc_bits->cb_index[10] |= ((*bitstreamPtr)>>12)&0xF; /* Bit 0..3 */ + enc_bits->cb_index[11] = ((*bitstreamPtr)>>4)&0xFF; /* Bit 4..11 */ + enc_bits->cb_index[12] |= ((*bitstreamPtr)>>3)&0x1; /* Bit 12 */ + enc_bits->cb_index[13] = ((*bitstreamPtr)<<5)&0xE0; /* Bit 13..15 */ + bitstreamPtr++; + /* 23:rd int16_t */ + enc_bits->cb_index[13] |= ((*bitstreamPtr)>>11)&0x1F;/* Bit 0..4 */ + enc_bits->cb_index[14] = ((*bitstreamPtr)>>3)&0xFF; /* Bit 5..12 */ + enc_bits->gain_index[3] |= ((*bitstreamPtr)>>1)&0x3; /* Bit 13..14 */ + enc_bits->gain_index[4] |= ((*bitstreamPtr)&0x1); /* Bit 15 */ + bitstreamPtr++; + /* 24:rd int16_t */ + enc_bits->gain_index[5] = ((*bitstreamPtr)>>13)&0x7; /* Bit 0..2 */ + enc_bits->gain_index[6] |= ((*bitstreamPtr)>>10)&0x7; /* Bit 3..5 */ + enc_bits->gain_index[7] |= ((*bitstreamPtr)>>8)&0x3; /* Bit 6..7 */ + enc_bits->gain_index[8] = ((*bitstreamPtr)>>5)&0x7; /* Bit 8..10 */ + enc_bits->gain_index[9] |= ((*bitstreamPtr)>>1)&0xF; /* Bit 11..14 */ + enc_bits->gain_index[10] |= ((*bitstreamPtr)<<2)&0x4; /* Bit 15 */ + bitstreamPtr++; + /* 25:rd int16_t */ + enc_bits->gain_index[10] |= ((*bitstreamPtr)>>14)&0x3; /* Bit 0..1 */ + enc_bits->gain_index[11] = ((*bitstreamPtr)>>11)&0x7; /* Bit 2..4 */ + enc_bits->gain_index[12] |= ((*bitstreamPtr)>>7)&0xF; /* Bit 5..8 */ + enc_bits->gain_index[13] |= ((*bitstreamPtr)>>4)&0x7; /* Bit 9..11 */ + enc_bits->gain_index[14] = ((*bitstreamPtr)>>1)&0x7; /* Bit 12..14 */ + } + /* Last bit should be zero, otherwise it's an "empty" frame */ + if (((*bitstreamPtr)&0x1) == 1) { + return(1); + } else { + return(0); + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/unpack_bits.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/unpack_bits.h new file mode 100644 index 0000000000..1a63280e6b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/unpack_bits.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_UnpackBits.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_UNPACK_BITS_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_UNPACK_BITS_H_ + +#include + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * unpacking of bits from bitstream, i.e., vector of bytes + *---------------------------------------------------------------*/ + +int16_t +WebRtcIlbcfix_UnpackBits(/* (o) "Empty" frame indicator */ + const uint16_t* + bitstream, /* (i) The packatized bitstream */ + iLBC_bits* + enc_bits, /* (o) Paramerers from bitstream */ + int16_t mode /* (i) Codec mode (20 or 30) */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.c new file mode 100644 index 0000000000..d9375fb995 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Vq3.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/vq3.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" + +/*----------------------------------------------------------------* + * vector quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Vq3( + int16_t *Xq, /* quantized vector (Q13) */ + int16_t *index, + int16_t *CB, /* codebook in Q13 */ + int16_t *X, /* vector to quantize (Q13) */ + int16_t n_cb + ){ + int16_t i, j; + int16_t pos, minindex=0; + int16_t tmp; + int32_t dist, mindist; + + pos = 0; + mindist = WEBRTC_SPL_WORD32_MAX; /* start value */ + + /* Find the codebook with the lowest square distance */ + for (j = 0; j < n_cb; j++) { + tmp = X[0] - CB[pos]; + dist = tmp * tmp; + for (i = 1; i < 3; i++) { + tmp = X[i] - CB[pos + i]; + dist += tmp * tmp; + } + + if (dist < mindist) { + mindist = dist; + minindex = j; + } + pos += 3; + } + + /* Store the quantized codebook and the index */ + for (i = 0; i < 3; i++) { + Xq[i] = CB[minindex*3 + i]; + } + *index = minindex; + +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.h new file mode 100644 index 0000000000..c946478a1a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Vq3.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_VQ3_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_VQ3_H_ + +#include + +/*----------------------------------------------------------------* + * Vector quantization of order 3 (based on MSE) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Vq3( + int16_t* Xq, /* (o) the quantized vector (Q13) */ + int16_t* index, /* (o) the quantization index */ + int16_t* CB, /* (i) the vector quantization codebook (Q13) */ + int16_t* X, /* (i) the vector to quantize (Q13) */ + int16_t n_cb /* (i) the number of vectors in the codebook */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.c new file mode 100644 index 0000000000..c9a65aec2a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Vq4.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/vq4.h" + +#include "modules/audio_coding/codecs/ilbc/constants.h" + +/*----------------------------------------------------------------* + * vector quantization + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Vq4( + int16_t *Xq, /* quantized vector (Q13) */ + int16_t *index, + int16_t *CB, /* codebook in Q13 */ + int16_t *X, /* vector to quantize (Q13) */ + int16_t n_cb + ){ + int16_t i, j; + int16_t pos, minindex=0; + int16_t tmp; + int32_t dist, mindist; + + pos = 0; + mindist = WEBRTC_SPL_WORD32_MAX; /* start value */ + + /* Find the codebook with the lowest square distance */ + for (j = 0; j < n_cb; j++) { + tmp = X[0] - CB[pos]; + dist = tmp * tmp; + for (i = 1; i < 4; i++) { + tmp = X[i] - CB[pos + i]; + dist += tmp * tmp; + } + + if (dist < mindist) { + mindist = dist; + minindex = j; + } + pos += 4; + } + + /* Store the quantized codebook and the index */ + for (i = 0; i < 4; i++) { + Xq[i] = CB[minindex*4 + i]; + } + *index = minindex; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.h new file mode 100644 index 0000000000..6d14830c03 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Vq4.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_VQ4_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_VQ4_H_ + +#include + +/*----------------------------------------------------------------* + * Vector quantization of order 4 (based on MSE) + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Vq4( + int16_t* Xq, /* (o) the quantized vector (Q13) */ + int16_t* index, /* (o) the quantization index */ + int16_t* CB, /* (i) the vector quantization codebook (Q13) */ + int16_t* X, /* (i) the vector to quantize (Q13) */ + int16_t n_cb /* (i) the number of vectors in the codebook */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.c new file mode 100644 index 0000000000..e82d167220 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Window32W32.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/window32_w32.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * window multiplication + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Window32W32( + int32_t *z, /* Output */ + int32_t *x, /* Input (same domain as Output)*/ + const int32_t *y, /* Q31 Window */ + size_t N /* length to process */ + ) { + size_t i; + int16_t x_low, x_hi, y_low, y_hi; + int16_t left_shifts; + int32_t temp; + + left_shifts = (int16_t)WebRtcSpl_NormW32(x[0]); + WebRtcSpl_VectorBitShiftW32(x, N, x, (int16_t)(-left_shifts)); + + + /* The double precision numbers use a special representation: + * w32 = hi<<16 + lo<<1 + */ + for (i = 0; i < N; i++) { + /* Extract higher bytes */ + x_hi = (int16_t)(x[i] >> 16); + y_hi = (int16_t)(y[i] >> 16); + + /* Extract lower bytes, defined as (w32 - hi<<16)>>1 */ + x_low = (int16_t)((x[i] - (x_hi << 16)) >> 1); + + y_low = (int16_t)((y[i] - (y_hi << 16)) >> 1); + + /* Calculate z by a 32 bit multiplication using both low and high from x and y */ + temp = ((x_hi * y_hi) << 1) + ((x_hi * y_low) >> 14); + + z[i] = temp + ((x_low * y_hi) >> 14); + } + + WebRtcSpl_VectorBitShiftW32(z, N, z, left_shifts); + + return; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.h new file mode 100644 index 0000000000..15d72c5ba2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_Window32W32.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_WINDOW32_W32_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_WINDOW32_W32_H_ + +#include +#include + +/*----------------------------------------------------------------* + * window multiplication + *---------------------------------------------------------------*/ + +void WebRtcIlbcfix_Window32W32(int32_t* z, /* Output */ + int32_t* x, /* Input (same domain as Output)*/ + const int32_t* y, /* Q31 Window */ + size_t N /* length to process */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c new file mode 100644 index 0000000000..9dc880b37e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_XcorrCoef.c + +******************************************************************/ + +#include "modules/audio_coding/codecs/ilbc/xcorr_coef.h" + +#include "modules/audio_coding/codecs/ilbc/defines.h" + +/*----------------------------------------------------------------* + * cross correlation which finds the optimal lag for the + * crossCorr*crossCorr/(energy) criteria + *---------------------------------------------------------------*/ + +size_t WebRtcIlbcfix_XcorrCoef( + int16_t *target, /* (i) first array */ + int16_t *regressor, /* (i) second array */ + size_t subl, /* (i) dimension arrays */ + size_t searchLen, /* (i) the search lenght */ + size_t offset, /* (i) samples offset between arrays */ + int16_t step /* (i) +1 or -1 */ + ){ + size_t k; + size_t maxlag; + int16_t pos; + int16_t max; + int16_t crossCorrScale, Energyscale; + int16_t crossCorrSqMod, crossCorrSqMod_Max; + int32_t crossCorr, Energy; + int16_t crossCorrmod, EnergyMod, EnergyMod_Max; + int16_t *tp, *rp; + int16_t *rp_beg, *rp_end; + int16_t totscale, totscale_max; + int16_t scalediff; + int32_t newCrit, maxCrit; + int shifts; + + /* Initializations, to make sure that the first one is selected */ + crossCorrSqMod_Max=0; + EnergyMod_Max=WEBRTC_SPL_WORD16_MAX; + totscale_max=-500; + maxlag=0; + pos=0; + + /* Find scale value and start position */ + if (step==1) { + max=WebRtcSpl_MaxAbsValueW16(regressor, subl + searchLen - 1); + rp_beg = regressor; + rp_end = regressor + subl; + } else { /* step==-1 */ + max = WebRtcSpl_MaxAbsValueW16(regressor - searchLen, subl + searchLen - 1); + rp_beg = regressor - 1; + rp_end = regressor + subl - 1; + } + + /* Introduce a scale factor on the Energy in int32_t in + order to make sure that the calculation does not + overflow */ + + if (max>5000) { + shifts=2; + } else { + shifts=0; + } + + /* Calculate the first energy, then do a +/- to get the other energies */ + Energy=WebRtcSpl_DotProductWithScale(regressor, regressor, subl, shifts); + + for (k=0;k0)&&(crossCorr>0)) { + + /* Put cross correlation and energy on 16 bit word */ + crossCorrScale=(int16_t)WebRtcSpl_NormW32(crossCorr)-16; + crossCorrmod=(int16_t)WEBRTC_SPL_SHIFT_W32(crossCorr, crossCorrScale); + Energyscale=(int16_t)WebRtcSpl_NormW32(Energy)-16; + EnergyMod=(int16_t)WEBRTC_SPL_SHIFT_W32(Energy, Energyscale); + + /* Square cross correlation and store upper int16_t */ + crossCorrSqMod = (int16_t)((crossCorrmod * crossCorrmod) >> 16); + + /* Calculate the total number of (dynamic) right shifts that have + been performed on (crossCorr*crossCorr)/energy + */ + totscale=Energyscale-(crossCorrScale<<1); + + /* Calculate the shift difference in order to be able to compare the two + (crossCorr*crossCorr)/energy in the same domain + */ + scalediff=totscale-totscale_max; + scalediff=WEBRTC_SPL_MIN(scalediff,31); + scalediff=WEBRTC_SPL_MAX(scalediff,-31); + + /* Compute the cross multiplication between the old best criteria + and the new one to be able to compare them without using a + division */ + + if (scalediff<0) { + newCrit = ((int32_t)crossCorrSqMod*EnergyMod_Max)>>(-scalediff); + maxCrit = ((int32_t)crossCorrSqMod_Max*EnergyMod); + } else { + newCrit = ((int32_t)crossCorrSqMod*EnergyMod_Max); + maxCrit = ((int32_t)crossCorrSqMod_Max*EnergyMod)>>scalediff; + } + + /* Store the new lag value if the new criteria is larger + than previous largest criteria */ + + if (newCrit > maxCrit) { + crossCorrSqMod_Max = crossCorrSqMod; + EnergyMod_Max = EnergyMod; + totscale_max = totscale; + maxlag = k; + } + } + pos+=step; + + /* Do a +/- to get the next energy */ + Energy += step * ((*rp_end * *rp_end - *rp_beg * *rp_beg) >> shifts); + rp_beg+=step; + rp_end+=step; + } + + return(maxlag+offset); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.h b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.h new file mode 100644 index 0000000000..3be5a296b5 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/****************************************************************** + + iLBC Speech Coder ANSI-C Source Code + + WebRtcIlbcfix_XcorrCoef.h + +******************************************************************/ + +#ifndef MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_XCORR_COEF_H_ +#define MODULES_AUDIO_CODING_CODECS_ILBC_MAIN_SOURCE_XCORR_COEF_H_ + +#include +#include + +/*----------------------------------------------------------------* + * cross correlation which finds the optimal lag for the + * crossCorr*crossCorr/(energy) criteria + *---------------------------------------------------------------*/ + +size_t WebRtcIlbcfix_XcorrCoef( + int16_t* target, /* (i) first array */ + int16_t* regressor, /* (i) second array */ + size_t subl, /* (i) dimension arrays */ + size_t searchLen, /* (i) the search lenght */ + size_t offset, /* (i) samples offset between arrays */ + int16_t step /* (i) +1 or -1 */ + ); + +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/bandwidth_info.h b/third_party/libwebrtc/modules/audio_coding/codecs/isac/bandwidth_info.h new file mode 100644 index 0000000000..c3830a5f7c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/bandwidth_info.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ + +#include + +typedef struct { + int in_use; + int32_t send_bw_avg; + int32_t send_max_delay_avg; + int16_t bottleneck_idx; + int16_t jitter_info; +} IsacBandwidthInfo; + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c new file mode 100644 index 0000000000..a4f297c5a1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#ifdef WEBRTC_ANDROID +#include +#endif + +#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h" +#include "modules/audio_coding/codecs/isac/main/source/isac_vad.h" + +static void WebRtcIsac_AllPoleFilter(double* InOut, + double* Coef, + size_t lengthInOut, + int orderCoef) { + /* the state of filter is assumed to be in InOut[-1] to InOut[-orderCoef] */ + double scal; + double sum; + size_t n; + int k; + + //if (fabs(Coef[0]-1.0)<0.001) { + if ( (Coef[0] > 0.9999) && (Coef[0] < 1.0001) ) + { + for(n = 0; n < lengthInOut; n++) + { + sum = Coef[1] * InOut[-1]; + for(k = 2; k <= orderCoef; k++){ + sum += Coef[k] * InOut[-k]; + } + *InOut++ -= sum; + } + } + else + { + scal = 1.0 / Coef[0]; + for(n=0;nbuffer, sizeof(double) * PITCH_WLPCBUFLEN); + memcpy(tmpbuffer+PITCH_WLPCBUFLEN, in, sizeof(double) * PITCH_FRAME_LEN); + memcpy(wfdata->buffer, tmpbuffer+PITCH_FRAME_LEN, sizeof(double) * PITCH_WLPCBUFLEN); + + dp=weoutbuf; + dp2=whoutbuf; + for (k=0;kweostate[k]; + *dp2++ = wfdata->whostate[k]; + opol[k]=0.0; + } + opol[0]=1.0; + opol[PITCH_WLPCORDER]=0.0; + weo=dp; + who=dp2; + + endpos=PITCH_WLPCBUFLEN + PITCH_SUBFRAME_LEN; + inp=tmpbuffer + PITCH_WLPCBUFLEN; + + for (n=0; nwindow[k]*tmpbuffer[start+k]; + } + + /* Get LPC polynomial */ + WebRtcIsac_AutoCorr(corr, ext, PITCH_WLPCWINLEN, PITCH_WLPCORDER); + corr[0]=1.01*corr[0]+1.0; /* White noise correction */ + WebRtcIsac_LevDurb(apol, rc, corr, PITCH_WLPCORDER); + WebRtcIsac_BwExpand(apolr, apol, rho, PITCH_WLPCORDER+1); + + /* Filtering */ + WebRtcIsac_ZeroPoleFilter(inp, apol, apolr, PITCH_SUBFRAME_LEN, PITCH_WLPCORDER, weo); + WebRtcIsac_ZeroPoleFilter(inp, apolr, opol, PITCH_SUBFRAME_LEN, PITCH_WLPCORDER, who); + + inp+=PITCH_SUBFRAME_LEN; + endpos+=PITCH_SUBFRAME_LEN; + weo+=PITCH_SUBFRAME_LEN; + who+=PITCH_SUBFRAME_LEN; + } + + /* Export filter states */ + for (k=0;kweostate[k]=weoutbuf[PITCH_FRAME_LEN+k]; + wfdata->whostate[k]=whoutbuf[PITCH_FRAME_LEN+k]; + } + + /* Export output data */ + memcpy(weiout, weoutbuf+PITCH_WLPCORDER, sizeof(double) * PITCH_FRAME_LEN); + memcpy(whiout, whoutbuf+PITCH_WLPCORDER, sizeof(double) * PITCH_FRAME_LEN); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.h b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.h new file mode 100644 index 0000000000..a747a7f549 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FILTER_FUNCTIONS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FILTER_FUNCTIONS_H_ + +#include + +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +void WebRtcIsac_AutoCorr(double* r, const double* x, size_t N, size_t order); + +void WebRtcIsac_WeightingFilter(const double* in, + double* weiout, + double* whiout, + WeightFiltstr* wfdata); + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FILTER_FUNCTIONS_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.c b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.c new file mode 100644 index 0000000000..57cf0c39da --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.c @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/main/source/isac_vad.h" + +#include + +void WebRtcIsac_InitPitchFilter(PitchFiltstr* pitchfiltdata) { + int k; + + for (k = 0; k < PITCH_BUFFSIZE; k++) { + pitchfiltdata->ubuf[k] = 0.0; + } + pitchfiltdata->ystate[0] = 0.0; + for (k = 1; k < (PITCH_DAMPORDER); k++) { + pitchfiltdata->ystate[k] = 0.0; + } + pitchfiltdata->oldlagp[0] = 50.0; + pitchfiltdata->oldgainp[0] = 0.0; +} + +static void WebRtcIsac_InitWeightingFilter(WeightFiltstr* wfdata) { + int k; + double t, dtmp, dtmp2, denum, denum2; + + for (k = 0; k < PITCH_WLPCBUFLEN; k++) + wfdata->buffer[k] = 0.0; + + for (k = 0; k < PITCH_WLPCORDER; k++) { + wfdata->istate[k] = 0.0; + wfdata->weostate[k] = 0.0; + wfdata->whostate[k] = 0.0; + } + + /* next part should be in Matlab, writing to a global table */ + t = 0.5; + denum = 1.0 / ((double)PITCH_WLPCWINLEN); + denum2 = denum * denum; + for (k = 0; k < PITCH_WLPCWINLEN; k++) { + dtmp = PITCH_WLPCASYM * t * denum + (1 - PITCH_WLPCASYM) * t * t * denum2; + dtmp *= 3.14159265; + dtmp2 = sin(dtmp); + wfdata->window[k] = dtmp2 * dtmp2; + t++; + } +} + +void WebRtcIsac_InitPitchAnalysis(PitchAnalysisStruct* State) { + int k; + + for (k = 0; k < PITCH_CORR_LEN2 + PITCH_CORR_STEP2 + PITCH_MAX_LAG / 2 - + PITCH_FRAME_LEN / 2 + 2; + k++) + State->dec_buffer[k] = 0.0; + for (k = 0; k < 2 * ALLPASSSECTIONS + 1; k++) + State->decimator_state[k] = 0.0; + for (k = 0; k < 2; k++) + State->hp_state[k] = 0.0; + for (k = 0; k < QLOOKAHEAD; k++) + State->whitened_buf[k] = 0.0; + for (k = 0; k < QLOOKAHEAD; k++) + State->inbuf[k] = 0.0; + + WebRtcIsac_InitPitchFilter(&(State->PFstr_wght)); + + WebRtcIsac_InitPitchFilter(&(State->PFstr)); + + WebRtcIsac_InitWeightingFilter(&(State->Wghtstr)); +} + +void WebRtcIsac_InitPreFilterbank(PreFiltBankstr* prefiltdata) { + int k; + + for (k = 0; k < QLOOKAHEAD; k++) { + prefiltdata->INLABUF1[k] = 0; + prefiltdata->INLABUF2[k] = 0; + + prefiltdata->INLABUF1_float[k] = 0; + prefiltdata->INLABUF2_float[k] = 0; + } + for (k = 0; k < 2 * (QORDER - 1); k++) { + prefiltdata->INSTAT1[k] = 0; + prefiltdata->INSTAT2[k] = 0; + prefiltdata->INSTATLA1[k] = 0; + prefiltdata->INSTATLA2[k] = 0; + + prefiltdata->INSTAT1_float[k] = 0; + prefiltdata->INSTAT2_float[k] = 0; + prefiltdata->INSTATLA1_float[k] = 0; + prefiltdata->INSTATLA2_float[k] = 0; + } + + /* High pass filter states */ + prefiltdata->HPstates[0] = 0.0; + prefiltdata->HPstates[1] = 0.0; + + prefiltdata->HPstates_float[0] = 0.0f; + prefiltdata->HPstates_float[1] = 0.0f; + + return; +} + +double WebRtcIsac_LevDurb(double* a, double* k, double* r, size_t order) { + const double LEVINSON_EPS = 1.0e-10; + + double sum, alpha; + size_t m, m_h, i; + alpha = 0; // warning -DH + a[0] = 1.0; + if (r[0] < LEVINSON_EPS) { /* if r[0] <= 0, set LPC coeff. to zero */ + for (i = 0; i < order; i++) { + k[i] = 0; + a[i + 1] = 0; + } + } else { + a[1] = k[0] = -r[1] / r[0]; + alpha = r[0] + r[1] * k[0]; + for (m = 1; m < order; m++) { + sum = r[m + 1]; + for (i = 0; i < m; i++) { + sum += a[i + 1] * r[m - i]; + } + k[m] = -sum / alpha; + alpha += k[m] * sum; + m_h = (m + 1) >> 1; + for (i = 0; i < m_h; i++) { + sum = a[i + 1] + k[m] * a[m - i]; + a[m - i] += k[m] * a[i + 1]; + a[i + 1] = sum; + } + a[m + 1] = k[m]; + } + } + return alpha; +} + +/* The upper channel all-pass filter factors */ +const float WebRtcIsac_kUpperApFactorsFloat[2] = {0.03470000000000f, + 0.38260000000000f}; + +/* The lower channel all-pass filter factors */ +const float WebRtcIsac_kLowerApFactorsFloat[2] = {0.15440000000000f, + 0.74400000000000f}; + +/* This function performs all-pass filtering--a series of first order all-pass + * sections are used to filter the input in a cascade manner. + * The input is overwritten!! + */ +void WebRtcIsac_AllPassFilter2Float(float* InOut, + const float* APSectionFactors, + int lengthInOut, + int NumberOfSections, + float* FilterState) { + int n, j; + float temp; + for (j = 0; j < NumberOfSections; j++) { + for (n = 0; n < lengthInOut; n++) { + temp = FilterState[j] + APSectionFactors[j] * InOut[n]; + FilterState[j] = -APSectionFactors[j] * temp + InOut[n]; + InOut[n] = temp; + } + } +} + +/* The number of composite all-pass filter factors */ +#define NUMBEROFCOMPOSITEAPSECTIONS 4 + +/* Function WebRtcIsac_SplitAndFilter + * This function creates low-pass and high-pass decimated versions of part of + the input signal, and part of the signal in the input 'lookahead buffer'. + + INPUTS: + in: a length FRAMESAMPLES array of input samples + prefiltdata: input data structure containing the filterbank states + and lookahead samples from the previous encoding + iteration. + OUTPUTS: + LP: a FRAMESAMPLES_HALF array of low-pass filtered samples that + have been phase equalized. The first QLOOKAHEAD samples are + based on the samples in the two prefiltdata->INLABUFx arrays + each of length QLOOKAHEAD. + The remaining FRAMESAMPLES_HALF-QLOOKAHEAD samples are based + on the first FRAMESAMPLES_HALF-QLOOKAHEAD samples of the input + array in[]. + HP: a FRAMESAMPLES_HALF array of high-pass filtered samples that + have been phase equalized. The first QLOOKAHEAD samples are + based on the samples in the two prefiltdata->INLABUFx arrays + each of length QLOOKAHEAD. + The remaining FRAMESAMPLES_HALF-QLOOKAHEAD samples are based + on the first FRAMESAMPLES_HALF-QLOOKAHEAD samples of the input + array in[]. + + LP_la: a FRAMESAMPLES_HALF array of low-pass filtered samples. + These samples are not phase equalized. They are computed + from the samples in the in[] array. + HP_la: a FRAMESAMPLES_HALF array of high-pass filtered samples + that are not phase equalized. They are computed from + the in[] vector. + prefiltdata: this input data structure's filterbank state and + lookahead sample buffers are updated for the next + encoding iteration. +*/ +void WebRtcIsac_SplitAndFilterFloat(float* pin, + float* LP, + float* HP, + double* LP_la, + double* HP_la, + PreFiltBankstr* prefiltdata) { + int k, n; + float CompositeAPFilterState[NUMBEROFCOMPOSITEAPSECTIONS]; + float ForTransform_CompositeAPFilterState[NUMBEROFCOMPOSITEAPSECTIONS]; + float ForTransform_CompositeAPFilterState2[NUMBEROFCOMPOSITEAPSECTIONS]; + float tempinoutvec[FRAMESAMPLES + MAX_AR_MODEL_ORDER]; + float tempin_ch1[FRAMESAMPLES + MAX_AR_MODEL_ORDER]; + float tempin_ch2[FRAMESAMPLES + MAX_AR_MODEL_ORDER]; + float in[FRAMESAMPLES]; + float ftmp; + + /* HPstcoeff_in = {a1, a2, b1 - b0 * a1, b2 - b0 * a2}; */ + static const float kHpStCoefInFloat[4] = { + -1.94895953203325f, 0.94984516000000f, -0.05101826139794f, + 0.05015484000000f}; + + /* The composite all-pass filter factors */ + static const float WebRtcIsac_kCompositeApFactorsFloat[4] = { + 0.03470000000000f, 0.15440000000000f, 0.38260000000000f, + 0.74400000000000f}; + + // The matrix for transforming the backward composite state to upper channel + // state. + static const float WebRtcIsac_kTransform1Float[8] = { + -0.00158678506084f, 0.00127157815343f, -0.00104805672709f, + 0.00084837248079f, 0.00134467983258f, -0.00107756549387f, + 0.00088814793277f, -0.00071893072525f}; + + // The matrix for transforming the backward composite state to lower channel + // state. + static const float WebRtcIsac_kTransform2Float[8] = { + -0.00170686041697f, 0.00136780109829f, -0.00112736532350f, + 0.00091257055385f, 0.00103094281812f, -0.00082615076557f, + 0.00068092756088f, -0.00055119165484f}; + + /* High pass filter */ + + for (k = 0; k < FRAMESAMPLES; k++) { + in[k] = pin[k] + kHpStCoefInFloat[2] * prefiltdata->HPstates_float[0] + + kHpStCoefInFloat[3] * prefiltdata->HPstates_float[1]; + ftmp = pin[k] - kHpStCoefInFloat[0] * prefiltdata->HPstates_float[0] - + kHpStCoefInFloat[1] * prefiltdata->HPstates_float[1]; + prefiltdata->HPstates_float[1] = prefiltdata->HPstates_float[0]; + prefiltdata->HPstates_float[0] = ftmp; + } + + /* First Channel */ + + /*initial state of composite filter is zero */ + for (k = 0; k < NUMBEROFCOMPOSITEAPSECTIONS; k++) { + CompositeAPFilterState[k] = 0.0; + } + /* put every other sample of input into a temporary vector in reverse + * (backward) order*/ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempinoutvec[k] = in[FRAMESAMPLES - 1 - 2 * k]; + } + + /* now all-pass filter the backwards vector. Output values overwrite the + * input vector. */ + WebRtcIsac_AllPassFilter2Float( + tempinoutvec, WebRtcIsac_kCompositeApFactorsFloat, FRAMESAMPLES_HALF, + NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + /* save the backwards filtered output for later forward filtering, + but write it in forward order*/ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempin_ch1[FRAMESAMPLES_HALF + QLOOKAHEAD - 1 - k] = tempinoutvec[k]; + } + + /* save the backwards filter state becaue it will be transformed + later into a forward state */ + for (k = 0; k < NUMBEROFCOMPOSITEAPSECTIONS; k++) { + ForTransform_CompositeAPFilterState[k] = CompositeAPFilterState[k]; + } + + /* now backwards filter the samples in the lookahead buffer. The samples were + placed there in the encoding of the previous frame. The output samples + overwrite the input samples */ + WebRtcIsac_AllPassFilter2Float( + prefiltdata->INLABUF1_float, WebRtcIsac_kCompositeApFactorsFloat, + QLOOKAHEAD, NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + /* save the output, but write it in forward order */ + /* write the lookahead samples for the next encoding iteration. Every other + sample at the end of the input frame is written in reverse order for the + lookahead length. Exported in the prefiltdata structure. */ + for (k = 0; k < QLOOKAHEAD; k++) { + tempin_ch1[QLOOKAHEAD - 1 - k] = prefiltdata->INLABUF1_float[k]; + prefiltdata->INLABUF1_float[k] = in[FRAMESAMPLES - 1 - 2 * k]; + } + + /* Second Channel. This is exactly like the first channel, except that the + even samples are now filtered instead (lower channel). */ + for (k = 0; k < NUMBEROFCOMPOSITEAPSECTIONS; k++) { + CompositeAPFilterState[k] = 0.0; + } + + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempinoutvec[k] = in[FRAMESAMPLES - 2 - 2 * k]; + } + + WebRtcIsac_AllPassFilter2Float( + tempinoutvec, WebRtcIsac_kCompositeApFactorsFloat, FRAMESAMPLES_HALF, + NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempin_ch2[FRAMESAMPLES_HALF + QLOOKAHEAD - 1 - k] = tempinoutvec[k]; + } + + for (k = 0; k < NUMBEROFCOMPOSITEAPSECTIONS; k++) { + ForTransform_CompositeAPFilterState2[k] = CompositeAPFilterState[k]; + } + + WebRtcIsac_AllPassFilter2Float( + prefiltdata->INLABUF2_float, WebRtcIsac_kCompositeApFactorsFloat, + QLOOKAHEAD, NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + for (k = 0; k < QLOOKAHEAD; k++) { + tempin_ch2[QLOOKAHEAD - 1 - k] = prefiltdata->INLABUF2_float[k]; + prefiltdata->INLABUF2_float[k] = in[FRAMESAMPLES - 2 - 2 * k]; + } + + /* Transform filter states from backward to forward */ + /*At this point, each of the states of the backwards composite filters for the + two channels are transformed into forward filtering states for the + corresponding forward channel filters. Each channel's forward filtering + state from the previous + encoding iteration is added to the transformed state to get a proper forward + state */ + + /* So the existing NUMBEROFCOMPOSITEAPSECTIONS x 1 (4x1) state vector is + multiplied by a NUMBEROFCHANNELAPSECTIONSxNUMBEROFCOMPOSITEAPSECTIONS (2x4) + transform matrix to get the new state that is added to the previous 2x1 + input state */ + + for (k = 0; k < NUMBEROFCHANNELAPSECTIONS; k++) { /* k is row variable */ + for (n = 0; n < NUMBEROFCOMPOSITEAPSECTIONS; + n++) { /* n is column variable */ + prefiltdata->INSTAT1_float[k] += + ForTransform_CompositeAPFilterState[n] * + WebRtcIsac_kTransform1Float[k * NUMBEROFCHANNELAPSECTIONS + n]; + prefiltdata->INSTAT2_float[k] += + ForTransform_CompositeAPFilterState2[n] * + WebRtcIsac_kTransform2Float[k * NUMBEROFCHANNELAPSECTIONS + n]; + } + } + + /*obtain polyphase components by forward all-pass filtering through each + * channel */ + /* the backward filtered samples are now forward filtered with the + * corresponding channel filters */ + /* The all pass filtering automatically updates the filter states which are + exported in the prefiltdata structure */ + WebRtcIsac_AllPassFilter2Float(tempin_ch1, WebRtcIsac_kUpperApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, + prefiltdata->INSTAT1_float); + WebRtcIsac_AllPassFilter2Float(tempin_ch2, WebRtcIsac_kLowerApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, + prefiltdata->INSTAT2_float); + + /* Now Construct low-pass and high-pass signals as combinations of polyphase + * components */ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + LP[k] = 0.5f * (tempin_ch1[k] + tempin_ch2[k]); /* low pass signal*/ + HP[k] = 0.5f * (tempin_ch1[k] - tempin_ch2[k]); /* high pass signal*/ + } + + /* Lookahead LP and HP signals */ + /* now create low pass and high pass signals of the input vector. However, no + backwards filtering is performed, and hence no phase equalization is + involved. Also, the input contains some samples that are lookahead samples. + The high pass and low pass signals that are created are used outside this + function for analysis (not encoding) purposes */ + + /* set up input */ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempin_ch1[k] = in[2 * k + 1]; + tempin_ch2[k] = in[2 * k]; + } + + /* the input filter states are passed in and updated by the all-pass filtering + routine and exported in the prefiltdata structure*/ + WebRtcIsac_AllPassFilter2Float(tempin_ch1, WebRtcIsac_kUpperApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, + prefiltdata->INSTATLA1_float); + WebRtcIsac_AllPassFilter2Float(tempin_ch2, WebRtcIsac_kLowerApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, + prefiltdata->INSTATLA2_float); + + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + LP_la[k] = (float)(0.5f * (tempin_ch1[k] + tempin_ch2[k])); /*low pass */ + HP_la[k] = (double)(0.5f * (tempin_ch1[k] - tempin_ch2[k])); /* high pass */ + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.h b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.h new file mode 100644 index 0000000000..1aecfc4046 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_VAD_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_VAD_H_ + +#include + +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +void WebRtcIsac_InitPitchFilter(PitchFiltstr* pitchfiltdata); +void WebRtcIsac_InitPitchAnalysis(PitchAnalysisStruct* state); +void WebRtcIsac_InitPreFilterbank(PreFiltBankstr* prefiltdata); + +double WebRtcIsac_LevDurb(double* a, double* k, double* r, size_t order); + +/* The number of all-pass filter factors in an upper or lower channel*/ +#define NUMBEROFCHANNELAPSECTIONS 2 + +/* The upper channel all-pass filter factors */ +extern const float WebRtcIsac_kUpperApFactorsFloat[2]; + +/* The lower channel all-pass filter factors */ +extern const float WebRtcIsac_kLowerApFactorsFloat[2]; + +void WebRtcIsac_AllPassFilter2Float(float* InOut, + const float* APSectionFactors, + int lengthInOut, + int NumberOfSections, + float* FilterState); +void WebRtcIsac_SplitAndFilterFloat(float* in, + float* LP, + float* HP, + double* LP_la, + double* HP_la, + PreFiltBankstr* prefiltdata); + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_VAD_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h new file mode 100644 index 0000000000..fe9afa4ba2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ + +#include + +#include "rtc_base/system/arch.h" + +#if defined(WEBRTC_POSIX) +#define WebRtcIsac_lrint lrint +#elif (defined(WEBRTC_ARCH_X86) && defined(WIN32)) +static __inline long int WebRtcIsac_lrint(double x_dbl) { + long int x_int; + + __asm { + fld x_dbl + fistp x_int + } + ; + + return x_int; +} +#else // Do a slow but correct implementation of lrint + +static __inline long int WebRtcIsac_lrint(double x_dbl) { + long int x_int; + x_int = (long int)floor(x_dbl + 0.499999999999); + return x_int; +} + +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c new file mode 100644 index 0000000000..8a19ac1710 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c @@ -0,0 +1,695 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h" + +#include +#include +#include +#ifdef WEBRTC_ANDROID +#include +#endif + +#include "modules/audio_coding/codecs/isac/main/source/filter_functions.h" +#include "modules/audio_coding/codecs/isac/main/source/pitch_filter.h" +#include "rtc_base/system/ignore_warnings.h" + +static const double kInterpolWin[8] = {-0.00067556028640, 0.02184247643159, -0.12203175715679, 0.60086484101160, + 0.60086484101160, -0.12203175715679, 0.02184247643159, -0.00067556028640}; + +/* interpolation filter */ +__inline static void IntrepolFilter(double *data_ptr, double *intrp) +{ + *intrp = kInterpolWin[0] * data_ptr[-3]; + *intrp += kInterpolWin[1] * data_ptr[-2]; + *intrp += kInterpolWin[2] * data_ptr[-1]; + *intrp += kInterpolWin[3] * data_ptr[0]; + *intrp += kInterpolWin[4] * data_ptr[1]; + *intrp += kInterpolWin[5] * data_ptr[2]; + *intrp += kInterpolWin[6] * data_ptr[3]; + *intrp += kInterpolWin[7] * data_ptr[4]; +} + + +/* 2D parabolic interpolation */ +/* probably some 0.5 factors can be eliminated, and the square-roots can be removed from the Cholesky fact. */ +__inline static void Intrpol2D(double T[3][3], double *x, double *y, double *peak_val) +{ + double c, b[2], A[2][2]; + double t1, t2, d; + double delta1, delta2; + + + // double T[3][3] = {{-1.25, -.25,-.25}, {-.25, .75, .75}, {-.25, .75, .75}}; + // should result in: delta1 = 0.5; delta2 = 0.0; peak_val = 1.0 + + c = T[1][1]; + b[0] = 0.5 * (T[1][2] + T[2][1] - T[0][1] - T[1][0]); + b[1] = 0.5 * (T[1][0] + T[2][1] - T[0][1] - T[1][2]); + A[0][1] = -0.5 * (T[0][1] + T[2][1] - T[1][0] - T[1][2]); + t1 = 0.5 * (T[0][0] + T[2][2]) - c; + t2 = 0.5 * (T[2][0] + T[0][2]) - c; + d = (T[0][1] + T[1][2] + T[1][0] + T[2][1]) - 4.0 * c - t1 - t2; + A[0][0] = -t1 - 0.5 * d; + A[1][1] = -t2 - 0.5 * d; + + /* deal with singularities or ill-conditioned cases */ + if ( (A[0][0] < 1e-7) || ((A[0][0] * A[1][1] - A[0][1] * A[0][1]) < 1e-7) ) { + *peak_val = T[1][1]; + return; + } + + /* Cholesky decomposition: replace A by upper-triangular factor */ + A[0][0] = sqrt(A[0][0]); + A[0][1] = A[0][1] / A[0][0]; + A[1][1] = sqrt(A[1][1] - A[0][1] * A[0][1]); + + /* compute [x; y] = -0.5 * inv(A) * b */ + t1 = b[0] / A[0][0]; + t2 = (b[1] - t1 * A[0][1]) / A[1][1]; + delta2 = t2 / A[1][1]; + delta1 = 0.5 * (t1 - delta2 * A[0][1]) / A[0][0]; + delta2 *= 0.5; + + /* limit norm */ + t1 = delta1 * delta1 + delta2 * delta2; + if (t1 > 1.0) { + delta1 /= t1; + delta2 /= t1; + } + + *peak_val = 0.5 * (b[0] * delta1 + b[1] * delta2) + c; + + *x += delta1; + *y += delta2; +} + + +static void PCorr(const double *in, double *outcorr) +{ + double sum, ysum, prod; + const double *x, *inptr; + int k, n; + + //ysum = 1e-6; /* use this with float (i.s.o. double)! */ + ysum = 1e-13; + sum = 0.0; + x = in + PITCH_MAX_LAG/2 + 2; + for (n = 0; n < PITCH_CORR_LEN2; n++) { + ysum += in[n] * in[n]; + sum += x[n] * in[n]; + } + + outcorr += PITCH_LAG_SPAN2 - 1; /* index of last element in array */ + *outcorr = sum / sqrt(ysum); + + for (k = 1; k < PITCH_LAG_SPAN2; k++) { + ysum -= in[k-1] * in[k-1]; + ysum += in[PITCH_CORR_LEN2 + k - 1] * in[PITCH_CORR_LEN2 + k - 1]; + sum = 0.0; + inptr = &in[k]; + prod = x[0] * inptr[0]; + for (n = 1; n < PITCH_CORR_LEN2; n++) { + sum += prod; + prod = x[n] * inptr[n]; + } + sum += prod; + outcorr--; + *outcorr = sum / sqrt(ysum); + } +} + +static void WebRtcIsac_AllpassFilterForDec(double* InOut, + const double* APSectionFactors, + size_t lengthInOut, + double* FilterState) { + // This performs all-pass filtering--a series of first order all-pass + // sections are used to filter the input in a cascade manner. + size_t n, j; + double temp; + for (j = 0; j < ALLPASSSECTIONS; j++) { + for (n = 0; n < lengthInOut; n += 2) { + temp = InOut[n]; // store input + InOut[n] = FilterState[j] + APSectionFactors[j] * temp; + FilterState[j] = -APSectionFactors[j] * InOut[n] + temp; + } + } +} + +static void WebRtcIsac_DecimateAllpass( + const double* in, + double* state_in, // array of size: 2*ALLPASSSECTIONS+1 + size_t N, // number of input samples + double* out) { // array of size N/2 + + static const double APupper[ALLPASSSECTIONS] = {0.0347, 0.3826}; + static const double APlower[ALLPASSSECTIONS] = {0.1544, 0.744}; + + size_t n; + double data_vec[PITCH_FRAME_LEN]; + + /* copy input */ + memcpy(data_vec + 1, in, sizeof(double) * (N - 1)); + + data_vec[0] = state_in[2 * ALLPASSSECTIONS]; // the z^(-1) state + state_in[2 * ALLPASSSECTIONS] = in[N - 1]; + + WebRtcIsac_AllpassFilterForDec(data_vec + 1, APupper, N, state_in); + WebRtcIsac_AllpassFilterForDec(data_vec, APlower, N, + state_in + ALLPASSSECTIONS); + + for (n = 0; n < N / 2; n++) + out[n] = data_vec[2 * n] + data_vec[2 * n + 1]; +} + +RTC_PUSH_IGNORING_WFRAME_LARGER_THAN() + +static void WebRtcIsac_InitializePitch(const double* in, + const double old_lag, + const double old_gain, + PitchAnalysisStruct* State, + double* lags) { + double buf_dec[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2+2]; + double ratio, log_lag, gain_bias; + double bias; + double corrvec1[PITCH_LAG_SPAN2]; + double corrvec2[PITCH_LAG_SPAN2]; + int m, k; + // Allocating 10 extra entries at the begining of the CorrSurf + double corrSurfBuff[10 + (2*PITCH_BW+3)*(PITCH_LAG_SPAN2+4)]; + double* CorrSurf[2*PITCH_BW+3]; + double *CorrSurfPtr1, *CorrSurfPtr2; + double LagWin[3] = {0.2, 0.5, 0.98}; + int ind1, ind2, peaks_ind, peak, max_ind; + int peaks[PITCH_MAX_NUM_PEAKS]; + double adj, gain_tmp; + double corr, corr_max; + double intrp_a, intrp_b, intrp_c, intrp_d; + double peak_vals[PITCH_MAX_NUM_PEAKS]; + double lags1[PITCH_MAX_NUM_PEAKS]; + double lags2[PITCH_MAX_NUM_PEAKS]; + double T[3][3]; + int row; + + for(k = 0; k < 2*PITCH_BW+3; k++) + { + CorrSurf[k] = &corrSurfBuff[10 + k * (PITCH_LAG_SPAN2+4)]; + } + /* reset CorrSurf matrix */ + memset(corrSurfBuff, 0, sizeof(double) * (10 + (2*PITCH_BW+3) * (PITCH_LAG_SPAN2+4))); + + //warnings -DH + max_ind = 0; + peak = 0; + + /* copy old values from state buffer */ + memcpy(buf_dec, State->dec_buffer, sizeof(double) * (PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2)); + + /* decimation; put result after the old values */ + WebRtcIsac_DecimateAllpass(in, State->decimator_state, PITCH_FRAME_LEN, + &buf_dec[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2]); + + /* low-pass filtering */ + for (k = PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2; k < PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2+2; k++) + buf_dec[k] += 0.75 * buf_dec[k-1] - 0.25 * buf_dec[k-2]; + + /* copy end part back into state buffer */ + memcpy(State->dec_buffer, buf_dec+PITCH_FRAME_LEN/2, sizeof(double) * (PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2)); + + /* compute correlation for first and second half of the frame */ + PCorr(buf_dec, corrvec1); + PCorr(buf_dec + PITCH_CORR_STEP2, corrvec2); + + /* bias towards pitch lag of previous frame */ + log_lag = log(0.5 * old_lag); + gain_bias = 4.0 * old_gain * old_gain; + if (gain_bias > 0.8) gain_bias = 0.8; + for (k = 0; k < PITCH_LAG_SPAN2; k++) + { + ratio = log((double) (k + (PITCH_MIN_LAG/2-2))) - log_lag; + bias = 1.0 + gain_bias * exp(-5.0 * ratio * ratio); + corrvec1[k] *= bias; + } + + /* taper correlation functions */ + for (k = 0; k < 3; k++) { + gain_tmp = LagWin[k]; + corrvec1[k] *= gain_tmp; + corrvec2[k] *= gain_tmp; + corrvec1[PITCH_LAG_SPAN2-1-k] *= gain_tmp; + corrvec2[PITCH_LAG_SPAN2-1-k] *= gain_tmp; + } + + corr_max = 0.0; + /* fill middle row of correlation surface */ + ind1 = 0; + ind2 = 0; + CorrSurfPtr1 = &CorrSurf[PITCH_BW][2]; + for (k = 0; k < PITCH_LAG_SPAN2; k++) { + corr = corrvec1[ind1++] + corrvec2[ind2++]; + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + } + /* fill first and last rows of correlation surface */ + ind1 = 0; + ind2 = PITCH_BW; + CorrSurfPtr1 = &CorrSurf[0][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW][PITCH_BW+2]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = 0.2 * ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + /* fill second and next to last rows of correlation surface */ + ind1 = 0; + ind2 = PITCH_BW-1; + CorrSurfPtr1 = &CorrSurf[1][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW-1][PITCH_BW+1]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW+1; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = 0.9 * ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + /* fill remainder of correlation surface */ + for (m = 2; m < PITCH_BW; m++) { + ind1 = 0; + ind2 = PITCH_BW - m; /* always larger than ind1 */ + CorrSurfPtr1 = &CorrSurf[m][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW-m][PITCH_BW+2-m]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW+m; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + } + + /* threshold value to qualify as a peak */ + corr_max *= 0.6; + + peaks_ind = 0; + /* find peaks */ + for (m = 1; m < PITCH_BW+1; m++) { + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + CorrSurfPtr1 = &CorrSurf[m][2]; + for (k = 2; k < PITCH_LAG_SPAN2-PITCH_BW-2+m; k++) { + corr = CorrSurfPtr1[k]; + if (corr > corr_max) { + if ( (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+5)]) && (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+4)]) ) { + if ( (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+4)]) && (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+5)]) ) { + /* found a peak; store index into matrix */ + peaks[peaks_ind++] = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + } + } + } + } + } + for (m = PITCH_BW+1; m < 2*PITCH_BW; m++) { + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + CorrSurfPtr1 = &CorrSurf[m][2]; + for (k = 2+m-PITCH_BW; k < PITCH_LAG_SPAN2-2; k++) { + corr = CorrSurfPtr1[k]; + if (corr > corr_max) { + if ( (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+5)]) && (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+4)]) ) { + if ( (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+4)]) && (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+5)]) ) { + /* found a peak; store index into matrix */ + peaks[peaks_ind++] = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + } + } + } + } + } + + if (peaks_ind > 0) { + /* examine each peak */ + CorrSurfPtr1 = &CorrSurf[0][0]; + for (k = 0; k < peaks_ind; k++) { + peak = peaks[k]; + + /* compute four interpolated values around current peak */ + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)], &intrp_a); + IntrepolFilter(&CorrSurfPtr1[peak - 1 ], &intrp_b); + IntrepolFilter(&CorrSurfPtr1[peak ], &intrp_c); + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)], &intrp_d); + + /* determine maximum of the interpolated values */ + corr = CorrSurfPtr1[peak]; + corr_max = intrp_a; + if (intrp_b > corr_max) corr_max = intrp_b; + if (intrp_c > corr_max) corr_max = intrp_c; + if (intrp_d > corr_max) corr_max = intrp_d; + + /* determine where the peak sits and fill a 3x3 matrix around it */ + row = peak / (PITCH_LAG_SPAN2+4); + lags1[k] = (double) ((peak - row * (PITCH_LAG_SPAN2+4)) + PITCH_MIN_LAG/2 - 4); + lags2[k] = (double) (lags1[k] + PITCH_BW - row); + if ( corr > corr_max ) { + T[0][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[2][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[1][1] = corr; + T[0][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + T[2][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + T[1][0] = intrp_a; + T[0][1] = intrp_b; + T[2][1] = intrp_c; + T[1][2] = intrp_d; + } else { + if (intrp_a == corr_max) { + lags1[k] -= 0.5; + lags2[k] += 0.5; + IntrepolFilter(&CorrSurfPtr1[peak - 2*(PITCH_LAG_SPAN2+5)], &T[0][0]); + IntrepolFilter(&CorrSurfPtr1[peak - (2*PITCH_LAG_SPAN2+9)], &T[2][0]); + T[1][1] = intrp_a; + T[0][2] = intrp_b; + T[2][2] = intrp_c; + T[1][0] = CorrSurfPtr1[peak - (2*PITCH_LAG_SPAN2+9)]; + T[0][1] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[2][1] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[1][2] = corr; + } else if (intrp_b == corr_max) { + lags1[k] -= 0.5; + lags2[k] -= 0.5; + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+6)], &T[0][0]); + T[2][0] = intrp_a; + T[1][1] = intrp_b; + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+3)], &T[0][2]); + T[2][2] = intrp_d; + T[1][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[0][1] = CorrSurfPtr1[peak - 1]; + T[2][1] = corr; + T[1][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + } else if (intrp_c == corr_max) { + lags1[k] += 0.5; + lags2[k] += 0.5; + T[0][0] = intrp_a; + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)], &T[2][0]); + T[1][1] = intrp_c; + T[0][2] = intrp_d; + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)], &T[2][2]); + T[1][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[0][1] = corr; + T[2][1] = CorrSurfPtr1[peak + 1]; + T[1][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + } else { + lags1[k] += 0.5; + lags2[k] -= 0.5; + T[0][0] = intrp_b; + T[2][0] = intrp_c; + T[1][1] = intrp_d; + IntrepolFilter(&CorrSurfPtr1[peak + 2*(PITCH_LAG_SPAN2+4)], &T[0][2]); + IntrepolFilter(&CorrSurfPtr1[peak + (2*PITCH_LAG_SPAN2+9)], &T[2][2]); + T[1][0] = corr; + T[0][1] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + T[2][1] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + T[1][2] = CorrSurfPtr1[peak + (2*PITCH_LAG_SPAN2+9)]; + } + } + + /* 2D parabolic interpolation gives more accurate lags and peak value */ + Intrpol2D(T, &lags1[k], &lags2[k], &peak_vals[k]); + } + + /* determine the highest peak, after applying a bias towards short lags */ + corr_max = 0.0; + for (k = 0; k < peaks_ind; k++) { + corr = peak_vals[k] * pow(PITCH_PEAK_DECAY, log(lags1[k] + lags2[k])); + if (corr > corr_max) { + corr_max = corr; + peak = k; + } + } + + lags1[peak] *= 2.0; + lags2[peak] *= 2.0; + + if (lags1[peak] < (double) PITCH_MIN_LAG) lags1[peak] = (double) PITCH_MIN_LAG; + if (lags2[peak] < (double) PITCH_MIN_LAG) lags2[peak] = (double) PITCH_MIN_LAG; + if (lags1[peak] > (double) PITCH_MAX_LAG) lags1[peak] = (double) PITCH_MAX_LAG; + if (lags2[peak] > (double) PITCH_MAX_LAG) lags2[peak] = (double) PITCH_MAX_LAG; + + /* store lags of highest peak in output array */ + lags[0] = lags1[peak]; + lags[1] = lags1[peak]; + lags[2] = lags2[peak]; + lags[3] = lags2[peak]; + } + else + { + row = max_ind / (PITCH_LAG_SPAN2+4); + lags1[0] = (double) ((max_ind - row * (PITCH_LAG_SPAN2+4)) + PITCH_MIN_LAG/2 - 4); + lags2[0] = (double) (lags1[0] + PITCH_BW - row); + + if (lags1[0] < (double) PITCH_MIN_LAG) lags1[0] = (double) PITCH_MIN_LAG; + if (lags2[0] < (double) PITCH_MIN_LAG) lags2[0] = (double) PITCH_MIN_LAG; + if (lags1[0] > (double) PITCH_MAX_LAG) lags1[0] = (double) PITCH_MAX_LAG; + if (lags2[0] > (double) PITCH_MAX_LAG) lags2[0] = (double) PITCH_MAX_LAG; + + /* store lags of highest peak in output array */ + lags[0] = lags1[0]; + lags[1] = lags1[0]; + lags[2] = lags2[0]; + lags[3] = lags2[0]; + } +} + +RTC_POP_IGNORING_WFRAME_LARGER_THAN() + +/* create weighting matrix by orthogonalizing a basis of polynomials of increasing order + * t = (0:4)'; + * A = [t.^0, t.^1, t.^2, t.^3, t.^4]; + * [Q, dummy] = qr(A); + * P.Weight = Q * diag([0, .1, .5, 1, 1]) * Q'; */ +static const double kWeight[5][5] = { + { 0.29714285714286, -0.30857142857143, -0.05714285714286, 0.05142857142857, 0.01714285714286}, + {-0.30857142857143, 0.67428571428571, -0.27142857142857, -0.14571428571429, 0.05142857142857}, + {-0.05714285714286, -0.27142857142857, 0.65714285714286, -0.27142857142857, -0.05714285714286}, + { 0.05142857142857, -0.14571428571429, -0.27142857142857, 0.67428571428571, -0.30857142857143}, + { 0.01714285714286, 0.05142857142857, -0.05714285714286, -0.30857142857143, 0.29714285714286} +}; + +/* second order high-pass filter */ +static void WebRtcIsac_Highpass(const double* in, + double* out, + double* state, + size_t N) { + /* create high-pass filter ocefficients + * z = 0.998 * exp(j*2*pi*35/8000); + * p = 0.94 * exp(j*2*pi*140/8000); + * HP_b = [1, -2*real(z), abs(z)^2]; + * HP_a = [1, -2*real(p), abs(p)^2]; */ + static const double a_coef[2] = { 1.86864659625574, -0.88360000000000}; + static const double b_coef[2] = {-1.99524591718270, 0.99600400000000}; + + size_t k; + + for (k=0; khp_state, PITCH_FRAME_LEN); + + /* copy from state into buffer */ + memcpy(Whitened, State->whitened_buf, sizeof(double) * QLOOKAHEAD); + + /* compute weighted and whitened signals */ + WebRtcIsac_WeightingFilter(HPin, &Weighted[0], &Whitened[QLOOKAHEAD], &(State->Wghtstr)); + + /* copy from buffer into state */ + memcpy(State->whitened_buf, Whitened+PITCH_FRAME_LEN, sizeof(double) * QLOOKAHEAD); + + old_lag = State->PFstr_wght.oldlagp[0]; + old_gain = State->PFstr_wght.oldgainp[0]; + + /* inital pitch estimate */ + WebRtcIsac_InitializePitch(Weighted, old_lag, old_gain, State, lags); + + + /* Iterative optimization of lags - to be done */ + + /* compute energy of whitened signal */ + nrg_wht = 0.0; + for (k = 0; k < PITCH_FRAME_LEN + QLOOKAHEAD; k++) + nrg_wht += Whitened[k] * Whitened[k]; + + + /* Iterative optimization of gains */ + + /* set weights for energy, gain fluctiation, and spectral gain penalty functions */ + Wnrg = 1.0 / nrg_wht; + Wgain = 0.005; + Wfluct = 3.0; + + /* set initial gains */ + for (k = 0; k < 4; k++) + gains[k] = PITCH_MAX_GAIN_06; + + /* two iterations should be enough */ + for (iter = 0; iter < 2; iter++) { + /* compute Jacobian of pre-filter output towards gains */ + WebRtcIsac_PitchfilterPre_gains(Whitened, out_G, out_dG, &(State->PFstr_wght), lags, gains); + + /* gradient and approximate Hessian (lower triangle) for minimizing the filter's output power */ + for (k = 0; k < 4; k++) { + tmp = 0.0; + for (n = 0; n < PITCH_FRAME_LEN + QLOOKAHEAD; n++) + tmp += out_G[n] * out_dG[k][n]; + grad[k] = tmp * Wnrg; + } + for (k = 0; k < 4; k++) { + for (m = 0; m <= k; m++) { + tmp = 0.0; + for (n = 0; n < PITCH_FRAME_LEN + QLOOKAHEAD; n++) + tmp += out_dG[m][n] * out_dG[k][n]; + H[k][m] = tmp * Wnrg; + } + } + + /* add gradient and Hessian (lower triangle) for dampening fast gain changes */ + for (k = 0; k < 4; k++) { + tmp = kWeight[k+1][0] * old_gain; + for (m = 0; m < 4; m++) + tmp += kWeight[k+1][m+1] * gains[m]; + grad[k] += tmp * Wfluct; + } + for (k = 0; k < 4; k++) { + for (m = 0; m <= k; m++) { + H[k][m] += kWeight[k+1][m+1] * Wfluct; + } + } + + /* add gradient and Hessian for dampening gain */ + for (k = 0; k < 3; k++) { + tmp = 1.0 / (1 - gains[k]); + grad[k] += tmp * tmp * Wgain; + H[k][k] += 2.0 * tmp * (tmp * tmp * Wgain); + } + tmp = 1.0 / (1 - gains[3]); + grad[3] += 1.33 * (tmp * tmp * Wgain); + H[3][3] += 2.66 * tmp * (tmp * tmp * Wgain); + + + /* compute Cholesky factorization of Hessian + * by overwritting the upper triangle; scale factors on diagonal + * (for non pc-platforms store the inverse of the diagonals seperately to minimize divisions) */ + H[0][1] = H[1][0] / H[0][0]; + H[0][2] = H[2][0] / H[0][0]; + H[0][3] = H[3][0] / H[0][0]; + H[1][1] -= H[0][0] * H[0][1] * H[0][1]; + H[1][2] = (H[2][1] - H[0][1] * H[2][0]) / H[1][1]; + H[1][3] = (H[3][1] - H[0][1] * H[3][0]) / H[1][1]; + H[2][2] -= H[0][0] * H[0][2] * H[0][2] + H[1][1] * H[1][2] * H[1][2]; + H[2][3] = (H[3][2] - H[0][2] * H[3][0] - H[1][2] * H[1][1] * H[1][3]) / H[2][2]; + H[3][3] -= H[0][0] * H[0][3] * H[0][3] + H[1][1] * H[1][3] * H[1][3] + H[2][2] * H[2][3] * H[2][3]; + + /* Compute update as delta_gains = -inv(H) * grad */ + /* copy and negate */ + for (k = 0; k < 4; k++) + dG[k] = -grad[k]; + /* back substitution */ + dG[1] -= dG[0] * H[0][1]; + dG[2] -= dG[0] * H[0][2] + dG[1] * H[1][2]; + dG[3] -= dG[0] * H[0][3] + dG[1] * H[1][3] + dG[2] * H[2][3]; + /* scale */ + for (k = 0; k < 4; k++) + dG[k] /= H[k][k]; + /* back substitution */ + dG[2] -= dG[3] * H[2][3]; + dG[1] -= dG[3] * H[1][3] + dG[2] * H[1][2]; + dG[0] -= dG[3] * H[0][3] + dG[2] * H[0][2] + dG[1] * H[0][1]; + + /* update gains and check range */ + for (k = 0; k < 4; k++) { + gains[k] += dG[k]; + if (gains[k] > PITCH_MAX_GAIN) + gains[k] = PITCH_MAX_GAIN; + else if (gains[k] < 0.0) + gains[k] = 0.0; + } + } + + /* update state for next frame */ + WebRtcIsac_PitchfilterPre(Whitened, out, &(State->PFstr_wght), lags, gains); + + /* concatenate previous input's end and current input */ + memcpy(inbuf, State->inbuf, sizeof(double) * QLOOKAHEAD); + memcpy(inbuf+QLOOKAHEAD, in, sizeof(double) * PITCH_FRAME_LEN); + + /* lookahead pitch filtering for masking analysis */ + WebRtcIsac_PitchfilterPre_la(inbuf, out, &(State->PFstr), lags, gains); + + /* store last part of input */ + for (k = 0; k < QLOOKAHEAD; k++) + State->inbuf[k] = inbuf[k + PITCH_FRAME_LEN]; +} + +RTC_POP_IGNORING_WFRAME_LARGER_THAN() diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h new file mode 100644 index 0000000000..4ab78c20ad --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_estimator.h + * + * Pitch functions + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ + +#include + +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +void WebRtcIsac_PitchAnalysis( + const double* in, /* PITCH_FRAME_LEN samples */ + double* out, /* PITCH_FRAME_LEN+QLOOKAHEAD samples */ + PitchAnalysisStruct* State, + double* lags, + double* gains); + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ */ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c new file mode 100644 index 0000000000..bf03dfff2e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c @@ -0,0 +1,388 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h" +#include "modules/audio_coding/codecs/isac/main/source/os_specific_inline.h" +#include "rtc_base/compile_assert_c.h" + +/* + * We are implementing the following filters; + * + * Pre-filtering: + * y(z) = x(z) + damper(z) * gain * (x(z) + y(z)) * z ^ (-lag); + * + * Post-filtering: + * y(z) = x(z) - damper(z) * gain * (x(z) + y(z)) * z ^ (-lag); + * + * Note that `lag` is a floating number so we perform an interpolation to + * obtain the correct `lag`. + * + */ + +static const double kDampFilter[PITCH_DAMPORDER] = {-0.07, 0.25, 0.64, 0.25, + -0.07}; + +/* interpolation coefficients; generated by design_pitch_filter.m */ +static const double kIntrpCoef[PITCH_FRACS][PITCH_FRACORDER] = { + {-0.02239172458614, 0.06653315052934, -0.16515880017569, 0.60701333734125, + 0.64671399919202, -0.20249000396417, 0.09926548334755, -0.04765933793109, + 0.01754159521746}, + {-0.01985640750434, 0.05816126837866, -0.13991265473714, 0.44560418147643, + 0.79117042386876, -0.20266133815188, 0.09585268418555, -0.04533310458084, + 0.01654127246314}, + {-0.01463300534216, 0.04229888475060, -0.09897034715253, 0.28284326017787, + 0.90385267956632, -0.16976950138649, 0.07704272393639, -0.03584218578311, + 0.01295781500709}, + {-0.00764851320885, 0.02184035544377, -0.04985561057281, 0.13083306574393, + 0.97545011664662, -0.10177807997561, 0.04400901776474, -0.02010737175166, + 0.00719783432422}, + {-0.00000000000000, 0.00000000000000, -0.00000000000001, 0.00000000000001, + 0.99999999999999, 0.00000000000001, -0.00000000000001, 0.00000000000000, + -0.00000000000000}, + {0.00719783432422, -0.02010737175166, 0.04400901776474, -0.10177807997562, + 0.97545011664663, 0.13083306574393, -0.04985561057280, 0.02184035544377, + -0.00764851320885}, + {0.01295781500710, -0.03584218578312, 0.07704272393640, -0.16976950138650, + 0.90385267956634, 0.28284326017785, -0.09897034715252, 0.04229888475059, + -0.01463300534216}, + {0.01654127246315, -0.04533310458085, 0.09585268418557, -0.20266133815190, + 0.79117042386878, 0.44560418147640, -0.13991265473712, 0.05816126837865, + -0.01985640750433} +}; + +/* + * Enumerating the operation of the filter. + * iSAC has 4 different pitch-filter which are very similar in their structure. + * + * kPitchFilterPre : In this mode the filter is operating as pitch + * pre-filter. This is used at the encoder. + * kPitchFilterPost : In this mode the filter is operating as pitch + * post-filter. This is the inverse of pre-filter and used + * in the decoder. + * kPitchFilterPreLa : This is, in structure, similar to pre-filtering but + * utilizing 3 millisecond lookahead. It is used to + * obtain the signal for LPC analysis. + * kPitchFilterPreGain : This is, in structure, similar to pre-filtering but + * differential changes in gain is considered. This is + * used to find the optimal gain. + */ +typedef enum { + kPitchFilterPre, kPitchFilterPost, kPitchFilterPreLa, kPitchFilterPreGain +} PitchFilterOperation; + +/* + * Structure with parameters used for pitch-filtering. + * buffer : a buffer where the sum of previous inputs and outputs + * are stored. + * damper_state : the state of the damping filter. The filter is defined by + * `kDampFilter`. + * interpol_coeff : pointer to a set of coefficient which are used to utilize + * fractional pitch by interpolation. + * gain : pitch-gain to be applied to the current segment of input. + * lag : pitch-lag for the current segment of input. + * lag_offset : the offset of lag w.r.t. current sample. + * sub_frame : sub-frame index, there are 4 pitch sub-frames in an iSAC + * frame. + * This specifies the usage of the filter. See + * 'PitchFilterOperation' for operational modes. + * num_samples : number of samples to be processed in each segment. + * index : index of the input and output sample. + * damper_state_dg : state of damping filter for different trial gains. + * gain_mult : differential changes to gain. + */ +typedef struct { + double buffer[PITCH_INTBUFFSIZE + QLOOKAHEAD]; + double damper_state[PITCH_DAMPORDER]; + const double *interpol_coeff; + double gain; + double lag; + int lag_offset; + + int sub_frame; + PitchFilterOperation mode; + int num_samples; + int index; + + double damper_state_dg[4][PITCH_DAMPORDER]; + double gain_mult[4]; +} PitchFilterParam; + +/********************************************************************** + * FilterSegment() + * Filter one segment, a quarter of a frame. + * + * Inputs + * in_data : pointer to the input signal of 30 ms at 8 kHz sample-rate. + * filter_param : pitch filter parameters. + * + * Outputs + * out_data : pointer to a buffer where the filtered signal is written to. + * out_dg : [only used in kPitchFilterPreGain] pointer to a buffer + * where the output of different gain values (differential + * change to gain) is written. + */ +static void FilterSegment(const double* in_data, PitchFilterParam* parameters, + double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD]) { + int n; + int m; + int j; + double sum; + double sum2; + /* Index of `parameters->buffer` where the output is written to. */ + int pos = parameters->index + PITCH_BUFFSIZE; + /* Index of `parameters->buffer` where samples are read for fractional-lag + * computation. */ + int pos_lag = pos - parameters->lag_offset; + + for (n = 0; n < parameters->num_samples; ++n) { + /* Shift low pass filter states. */ + for (m = PITCH_DAMPORDER - 1; m > 0; --m) { + parameters->damper_state[m] = parameters->damper_state[m - 1]; + } + /* Filter to get fractional pitch. */ + sum = 0.0; + for (m = 0; m < PITCH_FRACORDER; ++m) { + sum += parameters->buffer[pos_lag + m] * parameters->interpol_coeff[m]; + } + /* Multiply with gain. */ + parameters->damper_state[0] = parameters->gain * sum; + + if (parameters->mode == kPitchFilterPreGain) { + int lag_index = parameters->index - parameters->lag_offset; + int m_tmp = (lag_index < 0) ? -lag_index : 0; + /* Update the damper state for the new sample. */ + for (m = PITCH_DAMPORDER - 1; m > 0; --m) { + for (j = 0; j < 4; ++j) { + parameters->damper_state_dg[j][m] = + parameters->damper_state_dg[j][m - 1]; + } + } + + for (j = 0; j < parameters->sub_frame + 1; ++j) { + /* Filter for fractional pitch. */ + sum2 = 0.0; + for (m = PITCH_FRACORDER-1; m >= m_tmp; --m) { + /* `lag_index + m` is always larger than or equal to zero, see how + * m_tmp is computed. This is equivalent to assume samples outside + * `out_dg[j]` are zero. */ + sum2 += out_dg[j][lag_index + m] * parameters->interpol_coeff[m]; + } + /* Add the contribution of differential gain change. */ + parameters->damper_state_dg[j][0] = parameters->gain_mult[j] * sum + + parameters->gain * sum2; + } + + /* Filter with damping filter, and store the results. */ + for (j = 0; j < parameters->sub_frame + 1; ++j) { + sum = 0.0; + for (m = 0; m < PITCH_DAMPORDER; ++m) { + sum -= parameters->damper_state_dg[j][m] * kDampFilter[m]; + } + out_dg[j][parameters->index] = sum; + } + } + /* Filter with damping filter. */ + sum = 0.0; + for (m = 0; m < PITCH_DAMPORDER; ++m) { + sum += parameters->damper_state[m] * kDampFilter[m]; + } + + /* Subtract from input and update buffer. */ + out_data[parameters->index] = in_data[parameters->index] - sum; + parameters->buffer[pos] = in_data[parameters->index] + + out_data[parameters->index]; + + ++parameters->index; + ++pos; + ++pos_lag; + } + return; +} + +/* Update filter parameters based on the pitch-gains and pitch-lags. */ +static void Update(PitchFilterParam* parameters) { + double fraction; + int fraction_index; + /* Compute integer lag-offset. */ + parameters->lag_offset = WebRtcIsac_lrint(parameters->lag + PITCH_FILTDELAY + + 0.5); + /* Find correct set of coefficients for computing fractional pitch. */ + fraction = parameters->lag_offset - (parameters->lag + PITCH_FILTDELAY); + fraction_index = WebRtcIsac_lrint(PITCH_FRACS * fraction - 0.5); + parameters->interpol_coeff = kIntrpCoef[fraction_index]; + + if (parameters->mode == kPitchFilterPreGain) { + /* If in this mode make a differential change to pitch gain. */ + parameters->gain_mult[parameters->sub_frame] += 0.2; + if (parameters->gain_mult[parameters->sub_frame] > 1.0) { + parameters->gain_mult[parameters->sub_frame] = 1.0; + } + if (parameters->sub_frame > 0) { + parameters->gain_mult[parameters->sub_frame - 1] -= 0.2; + } + } +} + +/****************************************************************************** + * FilterFrame() + * Filter a frame of 30 millisecond, given pitch-lags and pitch-gains. + * + * Inputs + * in_data : pointer to the input signal of 30 ms at 8 kHz sample-rate. + * lags : pointer to pitch-lags, 4 lags per frame. + * gains : pointer to pitch-gians, 4 gains per frame. + * mode : defining the functionality of the filter. It takes the + * following values. + * kPitchFilterPre: Pitch pre-filter, used at encoder. + * kPitchFilterPost: Pitch post-filter, used at decoder. + * kPitchFilterPreLa: Pitch pre-filter with lookahead. + * kPitchFilterPreGain: Pitch pre-filter used to otain optimal + * pitch-gains. + * + * Outputs + * out_data : pointer to a buffer where the filtered signal is written to. + * out_dg : [only used in kPitchFilterPreGain] pointer to a buffer + * where the output of different gain values (differential + * change to gain) is written. + */ +static void FilterFrame(const double* in_data, PitchFiltstr* filter_state, + double* lags, double* gains, PitchFilterOperation mode, + double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD]) { + PitchFilterParam filter_parameters; + double gain_delta, lag_delta; + double old_lag, old_gain; + int n; + int m; + const double kEnhancer = 1.3; + + /* Set up buffer and states. */ + filter_parameters.index = 0; + filter_parameters.lag_offset = 0; + filter_parameters.mode = mode; + /* Copy states to local variables. */ + memcpy(filter_parameters.buffer, filter_state->ubuf, + sizeof(filter_state->ubuf)); + RTC_COMPILE_ASSERT(sizeof(filter_parameters.buffer) >= + sizeof(filter_state->ubuf)); + memset(filter_parameters.buffer + + sizeof(filter_state->ubuf) / sizeof(filter_state->ubuf[0]), + 0, sizeof(filter_parameters.buffer) - sizeof(filter_state->ubuf)); + memcpy(filter_parameters.damper_state, filter_state->ystate, + sizeof(filter_state->ystate)); + + if (mode == kPitchFilterPreGain) { + /* Clear buffers. */ + memset(filter_parameters.gain_mult, 0, sizeof(filter_parameters.gain_mult)); + memset(filter_parameters.damper_state_dg, 0, + sizeof(filter_parameters.damper_state_dg)); + for (n = 0; n < PITCH_SUBFRAMES; ++n) { + //memset(out_dg[n], 0, sizeof(double) * (PITCH_FRAME_LEN + QLOOKAHEAD)); + memset(out_dg[n], 0, sizeof(out_dg[n])); + } + } else if (mode == kPitchFilterPost) { + /* Make output more periodic. Negative sign is to change the structure + * of the filter. */ + for (n = 0; n < PITCH_SUBFRAMES; ++n) { + gains[n] *= -kEnhancer; + } + } + + old_lag = *filter_state->oldlagp; + old_gain = *filter_state->oldgainp; + + /* No interpolation if pitch lag step is big. */ + if ((lags[0] > (PITCH_UPSTEP * old_lag)) || + (lags[0] < (PITCH_DOWNSTEP * old_lag))) { + old_lag = lags[0]; + old_gain = gains[0]; + + if (mode == kPitchFilterPreGain) { + filter_parameters.gain_mult[0] = 1.0; + } + } + + filter_parameters.num_samples = PITCH_UPDATE; + for (m = 0; m < PITCH_SUBFRAMES; ++m) { + /* Set the sub-frame value. */ + filter_parameters.sub_frame = m; + /* Calculate interpolation steps for pitch-lag and pitch-gain. */ + lag_delta = (lags[m] - old_lag) / PITCH_GRAN_PER_SUBFRAME; + filter_parameters.lag = old_lag; + gain_delta = (gains[m] - old_gain) / PITCH_GRAN_PER_SUBFRAME; + filter_parameters.gain = old_gain; + /* Store for the next sub-frame. */ + old_lag = lags[m]; + old_gain = gains[m]; + + for (n = 0; n < PITCH_GRAN_PER_SUBFRAME; ++n) { + /* Step-wise interpolation of pitch gains and lags. As pitch-lag changes, + * some parameters of filter need to be update. */ + filter_parameters.gain += gain_delta; + filter_parameters.lag += lag_delta; + /* Update parameters according to new lag value. */ + Update(&filter_parameters); + /* Filter a segment of input. */ + FilterSegment(in_data, &filter_parameters, out_data, out_dg); + } + } + + if (mode != kPitchFilterPreGain) { + /* Export buffer and states. */ + memcpy(filter_state->ubuf, &filter_parameters.buffer[PITCH_FRAME_LEN], + sizeof(filter_state->ubuf)); + memcpy(filter_state->ystate, filter_parameters.damper_state, + sizeof(filter_state->ystate)); + + /* Store for the next frame. */ + *filter_state->oldlagp = old_lag; + *filter_state->oldgainp = old_gain; + } + + if ((mode == kPitchFilterPreGain) || (mode == kPitchFilterPreLa)) { + /* Filter the lookahead segment, this is treated as the last sub-frame. So + * set `pf_param` to last sub-frame. */ + filter_parameters.sub_frame = PITCH_SUBFRAMES - 1; + filter_parameters.num_samples = QLOOKAHEAD; + FilterSegment(in_data, &filter_parameters, out_data, out_dg); + } +} + +void WebRtcIsac_PitchfilterPre(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPre, out_data, NULL); +} + +void WebRtcIsac_PitchfilterPre_la(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPreLa, out_data, + NULL); +} + +void WebRtcIsac_PitchfilterPre_gains( + double* in_data, double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD], PitchFiltstr *pf_state, + double* lags, double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPreGain, out_data, + out_dg); +} + +void WebRtcIsac_PitchfilterPost(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPost, out_data, NULL); +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.h b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.h new file mode 100644 index 0000000000..9a232de87b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_FILTER_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_FILTER_H_ + +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +void WebRtcIsac_PitchfilterPre(double* indat, + double* outdat, + PitchFiltstr* pfp, + double* lags, + double* gains); + +void WebRtcIsac_PitchfilterPost(double* indat, + double* outdat, + PitchFiltstr* pfp, + double* lags, + double* gains); + +void WebRtcIsac_PitchfilterPre_la(double* indat, + double* outdat, + PitchFiltstr* pfp, + double* lags, + double* gains); + +void WebRtcIsac_PitchfilterPre_gains( + double* indat, + double* outdat, + double out_dG[][PITCH_FRAME_LEN + QLOOKAHEAD], + PitchFiltstr* pfp, + double* lags, + double* gains); + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_FILTER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/settings.h b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/settings.h new file mode 100644 index 0000000000..abce90c4f5 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/settings.h @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * settings.h + * + * Declaration of #defines used in the iSAC codec + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ + +/* sampling frequency (Hz) */ +#define FS 16000 + +/* number of samples per frame (either 320 (20ms), 480 (30ms) or 960 (60ms)) */ +#define INITIAL_FRAMESAMPLES 960 + +/* do not modify the following; this will have to be modified if we + * have a 20ms framesize option */ +/**********************************************************************/ +/* miliseconds */ +#define FRAMESIZE 30 +/* number of samples per frame processed in the encoder, 480 */ +#define FRAMESAMPLES 480 /* ((FRAMESIZE*FS)/1000) */ +#define FRAMESAMPLES_HALF 240 +#define FRAMESAMPLES_QUARTER 120 +/**********************************************************************/ + +/* max number of samples per frame (= 60 ms frame) */ +#define MAX_FRAMESAMPLES 960 +#define MAX_SWBFRAMESAMPLES (MAX_FRAMESAMPLES * 2) +/* number of samples per 10ms frame */ +#define FRAMESAMPLES_10ms ((10 * FS) / 1000) +#define SWBFRAMESAMPLES_10ms (FRAMESAMPLES_10ms * 2) +/* number of samples in 30 ms frame */ +#define FRAMESAMPLES_30ms 480 +/* number of subframes */ +#define SUBFRAMES 6 +/* length of a subframe */ +#define UPDATE 80 +/* length of half a subframe (low/high band) */ +#define HALF_SUBFRAMELEN (UPDATE / 2) +/* samples of look ahead (in a half-band, so actually + * half the samples of look ahead @ FS) */ +#define QLOOKAHEAD 24 /* 3 ms */ +/* order of AR model in spectral entropy coder */ +#define AR_ORDER 6 +/* order of LP model in spectral entropy coder */ +#define LP_ORDER 0 + +/* window length (masking analysis) */ +#define WINLEN 256 +/* order of low-band pole filter used to approximate masking curve */ +#define ORDERLO 12 +/* order of hi-band pole filter used to approximate masking curve */ +#define ORDERHI 6 + +#define UB_LPC_ORDER 4 +#define UB_LPC_VEC_PER_FRAME 2 +#define UB16_LPC_VEC_PER_FRAME 4 +#define UB_ACTIVE_SUBFRAMES 2 +#define UB_MAX_LPC_ORDER 6 +#define UB_INTERPOL_SEGMENTS 1 +#define UB16_INTERPOL_SEGMENTS 3 +#define LB_TOTAL_DELAY_SAMPLES 48 +enum ISACBandwidth { isac8kHz = 8, isac12kHz = 12, isac16kHz = 16 }; +enum ISACBand { + kIsacLowerBand = 0, + kIsacUpperBand12 = 1, + kIsacUpperBand16 = 2 +}; +enum IsacSamplingRate { kIsacWideband = 16, kIsacSuperWideband = 32 }; +#define UB_LPC_GAIN_DIM SUBFRAMES +#define FB_STATE_SIZE_WORD32 6 + +/* order for post_filter_bank */ +#define POSTQORDER 3 +/* order for pre-filterbank */ +#define QORDER 3 +/* another order */ +#define QORDER_ALL (POSTQORDER + QORDER - 1) +/* for decimator */ +#define ALLPASSSECTIONS 2 + +/* array size for byte stream in number of bytes. */ +/* The old maximum size still needed for the decoding */ +#define STREAM_SIZE_MAX 600 +#define STREAM_SIZE_MAX_30 200 /* 200 bytes=53.4 kbps @ 30 ms.framelength */ +#define STREAM_SIZE_MAX_60 400 /* 400 bytes=53.4 kbps @ 60 ms.framelength */ + +/* storage size for bit counts */ +#define BIT_COUNTER_SIZE 30 +/* maximum order of any AR model or filter */ +#define MAX_AR_MODEL_ORDER 12 // 50 + +/* For pitch analysis */ +#define PITCH_FRAME_LEN (FRAMESAMPLES_HALF) /* 30 ms */ +#define PITCH_MAX_LAG 140 /* 57 Hz */ +#define PITCH_MIN_LAG 20 /* 400 Hz */ +#define PITCH_MAX_GAIN 0.45 +#define PITCH_MAX_GAIN_06 0.27 /* PITCH_MAX_GAIN*0.6 */ +#define PITCH_MAX_GAIN_Q12 1843 +#define PITCH_LAG_SPAN2 (PITCH_MAX_LAG / 2 - PITCH_MIN_LAG / 2 + 5) +#define PITCH_CORR_LEN2 60 /* 15 ms */ +#define PITCH_CORR_STEP2 (PITCH_FRAME_LEN / 4) +#define PITCH_BW 11 /* half the band width of correlation surface */ +#define PITCH_SUBFRAMES 4 +#define PITCH_GRAN_PER_SUBFRAME 5 +#define PITCH_SUBFRAME_LEN (PITCH_FRAME_LEN / PITCH_SUBFRAMES) +#define PITCH_UPDATE (PITCH_SUBFRAME_LEN / PITCH_GRAN_PER_SUBFRAME) +/* maximum number of peaks to be examined in correlation surface */ +#define PITCH_MAX_NUM_PEAKS 10 +#define PITCH_PEAK_DECAY 0.85 +/* For weighting filter */ +#define PITCH_WLPCORDER 6 +#define PITCH_WLPCWINLEN PITCH_FRAME_LEN +#define PITCH_WLPCASYM 0.3 /* asymmetry parameter */ +#define PITCH_WLPCBUFLEN PITCH_WLPCWINLEN +/* For pitch filter */ +/* Extra 50 for fraction and LP filters */ +#define PITCH_BUFFSIZE (PITCH_MAX_LAG + 50) +#define PITCH_INTBUFFSIZE (PITCH_FRAME_LEN + PITCH_BUFFSIZE) +/* Max rel. step for interpolation */ +#define PITCH_UPSTEP 1.5 +/* Max rel. step for interpolation */ +#define PITCH_DOWNSTEP 0.67 +#define PITCH_FRACS 8 +#define PITCH_FRACORDER 9 +#define PITCH_DAMPORDER 5 +#define PITCH_FILTDELAY 1.5f +/* stepsize for quantization of the pitch Gain */ +#define PITCH_GAIN_STEPSIZE 0.125 + +/* Order of high pass filter */ +#define HPORDER 2 + +/* some mathematical constants */ +/* log2(exp) */ +#define LOG2EXP 1.44269504088896 +#define PI 3.14159265358979 + +/* Maximum number of iterations allowed to limit payload size */ +#define MAX_PAYLOAD_LIMIT_ITERATION 5 + +/* Redundant Coding */ +#define RCU_BOTTLENECK_BPS 16000 +#define RCU_TRANSCODING_SCALE 0.40f +#define RCU_TRANSCODING_SCALE_INVERSE 2.5f + +#define RCU_TRANSCODING_SCALE_UB 0.50f +#define RCU_TRANSCODING_SCALE_UB_INVERSE 2.0f + +/* Define Error codes */ +/* 6000 General */ +#define ISAC_MEMORY_ALLOCATION_FAILED 6010 +#define ISAC_MODE_MISMATCH 6020 +#define ISAC_DISALLOWED_BOTTLENECK 6030 +#define ISAC_DISALLOWED_FRAME_LENGTH 6040 +#define ISAC_UNSUPPORTED_SAMPLING_FREQUENCY 6050 + +/* 6200 Bandwidth estimator */ +#define ISAC_RANGE_ERROR_BW_ESTIMATOR 6240 +/* 6400 Encoder */ +#define ISAC_ENCODER_NOT_INITIATED 6410 +#define ISAC_DISALLOWED_CODING_MODE 6420 +#define ISAC_DISALLOWED_FRAME_MODE_ENCODER 6430 +#define ISAC_DISALLOWED_BITSTREAM_LENGTH 6440 +#define ISAC_PAYLOAD_LARGER_THAN_LIMIT 6450 +#define ISAC_DISALLOWED_ENCODER_BANDWIDTH 6460 +/* 6600 Decoder */ +#define ISAC_DECODER_NOT_INITIATED 6610 +#define ISAC_EMPTY_PACKET 6620 +#define ISAC_DISALLOWED_FRAME_MODE_DECODER 6630 +#define ISAC_RANGE_ERROR_DECODE_FRAME_LENGTH 6640 +#define ISAC_RANGE_ERROR_DECODE_BANDWIDTH 6650 +#define ISAC_RANGE_ERROR_DECODE_PITCH_GAIN 6660 +#define ISAC_RANGE_ERROR_DECODE_PITCH_LAG 6670 +#define ISAC_RANGE_ERROR_DECODE_LPC 6680 +#define ISAC_RANGE_ERROR_DECODE_SPECTRUM 6690 +#define ISAC_LENGTH_MISMATCH 6730 +#define ISAC_RANGE_ERROR_DECODE_BANDWITH 6740 +#define ISAC_DISALLOWED_BANDWIDTH_MODE_DECODER 6750 +#define ISAC_DISALLOWED_LPC_MODEL 6760 +/* 6800 Call setup formats */ +#define ISAC_INCOMPATIBLE_FORMATS 6810 + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ */ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/structs.h b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/structs.h new file mode 100644 index 0000000000..6861ca42bd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/structs.h @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * structs.h + * + * This header file contains all the structs used in the ISAC codec + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ + +#include "modules/audio_coding/codecs/isac/bandwidth_info.h" +#include "modules/audio_coding/codecs/isac/main/source/settings.h" +#include "modules/third_party/fft/fft.h" + +typedef struct Bitstreamstruct { + uint8_t stream[STREAM_SIZE_MAX]; + uint32_t W_upper; + uint32_t streamval; + uint32_t stream_index; + +} Bitstr; + +typedef struct { + double DataBufferLo[WINLEN]; + double DataBufferHi[WINLEN]; + + double CorrBufLo[ORDERLO + 1]; + double CorrBufHi[ORDERHI + 1]; + + float PreStateLoF[ORDERLO + 1]; + float PreStateLoG[ORDERLO + 1]; + float PreStateHiF[ORDERHI + 1]; + float PreStateHiG[ORDERHI + 1]; + float PostStateLoF[ORDERLO + 1]; + float PostStateLoG[ORDERLO + 1]; + float PostStateHiF[ORDERHI + 1]; + float PostStateHiG[ORDERHI + 1]; + + double OldEnergy; + +} MaskFiltstr; + +typedef struct { + // state vectors for each of the two analysis filters + double INSTAT1[2 * (QORDER - 1)]; + double INSTAT2[2 * (QORDER - 1)]; + double INSTATLA1[2 * (QORDER - 1)]; + double INSTATLA2[2 * (QORDER - 1)]; + double INLABUF1[QLOOKAHEAD]; + double INLABUF2[QLOOKAHEAD]; + + float INSTAT1_float[2 * (QORDER - 1)]; + float INSTAT2_float[2 * (QORDER - 1)]; + float INSTATLA1_float[2 * (QORDER - 1)]; + float INSTATLA2_float[2 * (QORDER - 1)]; + float INLABUF1_float[QLOOKAHEAD]; + float INLABUF2_float[QLOOKAHEAD]; + + /* High pass filter */ + double HPstates[HPORDER]; + float HPstates_float[HPORDER]; + +} PreFiltBankstr; + +typedef struct { + // state vectors for each of the two analysis filters + double STATE_0_LOWER[2 * POSTQORDER]; + double STATE_0_UPPER[2 * POSTQORDER]; + + /* High pass filter */ + double HPstates1[HPORDER]; + double HPstates2[HPORDER]; + + float STATE_0_LOWER_float[2 * POSTQORDER]; + float STATE_0_UPPER_float[2 * POSTQORDER]; + + float HPstates1_float[HPORDER]; + float HPstates2_float[HPORDER]; + +} PostFiltBankstr; + +typedef struct { + // data buffer for pitch filter + double ubuf[PITCH_BUFFSIZE]; + + // low pass state vector + double ystate[PITCH_DAMPORDER]; + + // old lag and gain + double oldlagp[1]; + double oldgainp[1]; + +} PitchFiltstr; + +typedef struct { + // data buffer + double buffer[PITCH_WLPCBUFLEN]; + + // state vectors + double istate[PITCH_WLPCORDER]; + double weostate[PITCH_WLPCORDER]; + double whostate[PITCH_WLPCORDER]; + + // LPC window -> should be a global array because constant + double window[PITCH_WLPCWINLEN]; + +} WeightFiltstr; + +typedef struct { + // for inital estimator + double dec_buffer[PITCH_CORR_LEN2 + PITCH_CORR_STEP2 + PITCH_MAX_LAG / 2 - + PITCH_FRAME_LEN / 2 + 2]; + double decimator_state[2 * ALLPASSSECTIONS + 1]; + double hp_state[2]; + + double whitened_buf[QLOOKAHEAD]; + + double inbuf[QLOOKAHEAD]; + + PitchFiltstr PFstr_wght; + PitchFiltstr PFstr; + WeightFiltstr Wghtstr; + +} PitchAnalysisStruct; + +/* Have instance of struct together with other iSAC structs */ +typedef struct { + /* Previous frame length (in ms) */ + int32_t prev_frame_length; + + /* Previous RTP timestamp from received + packet (in samples relative beginning) */ + int32_t prev_rec_rtp_number; + + /* Send timestamp for previous packet (in ms using timeGetTime()) */ + uint32_t prev_rec_send_ts; + + /* Arrival time for previous packet (in ms using timeGetTime()) */ + uint32_t prev_rec_arr_ts; + + /* rate of previous packet, derived from RTP timestamps (in bits/s) */ + float prev_rec_rtp_rate; + + /* Time sinse the last update of the BN estimate (in ms) */ + uint32_t last_update_ts; + + /* Time sinse the last reduction (in ms) */ + uint32_t last_reduction_ts; + + /* How many times the estimate was update in the beginning */ + int32_t count_tot_updates_rec; + + /* The estimated bottle neck rate from there to here (in bits/s) */ + int32_t rec_bw; + float rec_bw_inv; + float rec_bw_avg; + float rec_bw_avg_Q; + + /* The estimated mean absolute jitter value, + as seen on this side (in ms) */ + float rec_jitter; + float rec_jitter_short_term; + float rec_jitter_short_term_abs; + float rec_max_delay; + float rec_max_delay_avg_Q; + + /* (assumed) bitrate for headers (bps) */ + float rec_header_rate; + + /* The estimated bottle neck rate from here to there (in bits/s) */ + float send_bw_avg; + + /* The estimated mean absolute jitter value, as seen on + the other siee (in ms) */ + float send_max_delay_avg; + + // number of packets received since last update + int num_pkts_rec; + + int num_consec_rec_pkts_over_30k; + + // flag for marking that a high speed network has been + // detected downstream + int hsn_detect_rec; + + int num_consec_snt_pkts_over_30k; + + // flag for marking that a high speed network has + // been detected upstream + int hsn_detect_snd; + + uint32_t start_wait_period; + + int in_wait_period; + + int change_to_WB; + + uint32_t senderTimestamp; + uint32_t receiverTimestamp; + // enum IsacSamplingRate incomingStreamSampFreq; + uint16_t numConsecLatePkts; + float consecLatency; + int16_t inWaitLatePkts; + + IsacBandwidthInfo external_bw_info; +} BwEstimatorstr; + +typedef struct { + /* boolean, flags if previous packet exceeded B.N. */ + int PrevExceed; + /* ms */ + int ExceedAgo; + /* packets left to send in current burst */ + int BurstCounter; + /* packets */ + int InitCounter; + /* ms remaining in buffer when next packet will be sent */ + double StillBuffered; + +} RateModel; + +/* The following strutc is used to store data from encoding, to make it + fast and easy to construct a new bitstream with a different Bandwidth + estimate. All values (except framelength and minBytes) is double size to + handle 60 ms of data. +*/ +typedef struct { + /* Used to keep track of if it is first or second part of 60 msec packet */ + int startIdx; + + /* Frame length in samples */ + int16_t framelength; + + /* Pitch Gain */ + int pitchGain_index[2]; + + /* Pitch Lag */ + double meanGain[2]; + int pitchIndex[PITCH_SUBFRAMES * 2]; + + /* LPC */ + int LPCindex_s[108 * 2]; /* KLT_ORDER_SHAPE = 108 */ + int LPCindex_g[12 * 2]; /* KLT_ORDER_GAIN = 12 */ + double LPCcoeffs_lo[(ORDERLO + 1) * SUBFRAMES * 2]; + double LPCcoeffs_hi[(ORDERHI + 1) * SUBFRAMES * 2]; + + /* Encode Spec */ + int16_t fre[FRAMESAMPLES]; + int16_t fim[FRAMESAMPLES]; + int16_t AvgPitchGain[2]; + + /* Used in adaptive mode only */ + int minBytes; + +} IsacSaveEncoderData; + +typedef struct { + int indexLPCShape[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + double lpcGain[SUBFRAMES << 1]; + int lpcGainIndex[SUBFRAMES << 1]; + + Bitstr bitStreamObj; + + int16_t realFFT[FRAMESAMPLES_HALF]; + int16_t imagFFT[FRAMESAMPLES_HALF]; +} ISACUBSaveEncDataStruct; + +typedef struct { + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PreFiltBankstr prefiltbankstr_obj; + PitchFiltstr pitchfiltstr_obj; + PitchAnalysisStruct pitchanalysisstr_obj; + FFTstr fftstr_obj; + IsacSaveEncoderData SaveEnc_obj; + + int buffer_index; + int16_t current_framesamples; + + float data_buffer_float[FRAMESAMPLES_30ms]; + + int frame_nb; + double bottleneck; + int16_t new_framelength; + double s2nr; + + /* Maximum allowed number of bits for a 30 msec packet */ + int16_t payloadLimitBytes30; + /* Maximum allowed number of bits for a 30 msec packet */ + int16_t payloadLimitBytes60; + /* Maximum allowed number of bits for both 30 and 60 msec packet */ + int16_t maxPayloadBytes; + /* Maximum allowed rate in bytes per 30 msec packet */ + int16_t maxRateInBytes; + + /*--- + If set to 1 iSAC will not adapt the frame-size, if used in + channel-adaptive mode. The initial value will be used for all rates. + ---*/ + int16_t enforceFrameSize; + + /*----- + This records the BWE index the encoder injected into the bit-stream. + It will be used in RCU. The same BWE index of main payload will be in + the redundant payload. We can not retrieve it from BWE because it is + a recursive procedure (WebRtcIsac_GetDownlinkBwJitIndexImpl) and has to be + called only once per each encode. + -----*/ + int16_t lastBWIdx; +} ISACLBEncStruct; + +typedef struct { + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PreFiltBankstr prefiltbankstr_obj; + FFTstr fftstr_obj; + ISACUBSaveEncDataStruct SaveEnc_obj; + + int buffer_index; + float data_buffer_float[MAX_FRAMESAMPLES + LB_TOTAL_DELAY_SAMPLES]; + double bottleneck; + /* Maximum allowed number of bits for a 30 msec packet */ + // int16_t payloadLimitBytes30; + /* Maximum allowed number of bits for both 30 and 60 msec packet */ + // int16_t maxPayloadBytes; + int16_t maxPayloadSizeBytes; + + double lastLPCVec[UB_LPC_ORDER]; + int16_t numBytesUsed; + int16_t lastJitterInfo; +} ISACUBEncStruct; + +typedef struct { + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PostFiltBankstr postfiltbankstr_obj; + PitchFiltstr pitchfiltstr_obj; + FFTstr fftstr_obj; + +} ISACLBDecStruct; + +typedef struct { + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PostFiltBankstr postfiltbankstr_obj; + FFTstr fftstr_obj; + +} ISACUBDecStruct; + +typedef struct { + ISACLBEncStruct ISACencLB_obj; + ISACLBDecStruct ISACdecLB_obj; +} ISACLBStruct; + +typedef struct { + ISACUBEncStruct ISACencUB_obj; + ISACUBDecStruct ISACdecUB_obj; +} ISACUBStruct; + +/* + This struct is used to take a snapshot of the entropy coder and LPC gains + right before encoding LPC gains. This allows us to go back to that state + if we like to limit the payload size. +*/ +typedef struct { + /* 6 lower-band & 6 upper-band */ + double loFiltGain[SUBFRAMES]; + double hiFiltGain[SUBFRAMES]; + /* Upper boundary of interval W */ + uint32_t W_upper; + uint32_t streamval; + /* Index to the current position in bytestream */ + uint32_t stream_index; + uint8_t stream[3]; +} transcode_obj; + +typedef struct { + // TODO(kwiberg): The size of these tables could be reduced by storing floats + // instead of doubles, and by making use of the identity cos(x) = + // sin(x+pi/2). They could also be made global constants that we fill in at + // compile time. + double costab1[FRAMESAMPLES_HALF]; + double sintab1[FRAMESAMPLES_HALF]; + double costab2[FRAMESAMPLES_QUARTER]; + double sintab2[FRAMESAMPLES_QUARTER]; +} TransformTables; + +typedef struct { + // lower-band codec instance + ISACLBStruct instLB; + // upper-band codec instance + ISACUBStruct instUB; + + // Bandwidth Estimator and model for the rate. + BwEstimatorstr bwestimator_obj; + RateModel rate_data_obj; + double MaxDelay; + + /* 0 = adaptive; 1 = instantaneous */ + int16_t codingMode; + + // overall bottleneck of the codec + int32_t bottleneck; + + // QMF Filter state + int32_t analysisFBState1[FB_STATE_SIZE_WORD32]; + int32_t analysisFBState2[FB_STATE_SIZE_WORD32]; + int32_t synthesisFBState1[FB_STATE_SIZE_WORD32]; + int32_t synthesisFBState2[FB_STATE_SIZE_WORD32]; + + // Error Code + int16_t errorCode; + + // bandwidth of the encoded audio 8, 12 or 16 kHz + enum ISACBandwidth bandwidthKHz; + // Sampling rate of audio, encoder and decode, 8 or 16 kHz + enum IsacSamplingRate encoderSamplingRateKHz; + enum IsacSamplingRate decoderSamplingRateKHz; + // Flag to keep track of initializations, lower & upper-band + // encoder and decoder. + int16_t initFlag; + + // Flag to to indicate signal bandwidth switch + int16_t resetFlag_8kHz; + + // Maximum allowed rate, measured in Bytes per 30 ms. + int16_t maxRateBytesPer30Ms; + // Maximum allowed payload-size, measured in Bytes. + int16_t maxPayloadSizeBytes; + /* The expected sampling rate of the input signal. Valid values are 16000 + * and 32000. This is not the operation sampling rate of the codec. */ + uint16_t in_sample_rate_hz; + + // Trig tables for WebRtcIsac_Time2Spec and WebRtcIsac_Spec2time. + TransformTables transform_tables; +} ISACMainStruct; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ */ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc b/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc new file mode 100644 index 0000000000..dacf325082 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" + +#include +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +LegacyEncodedAudioFrame::LegacyEncodedAudioFrame(AudioDecoder* decoder, + rtc::Buffer&& payload) + : decoder_(decoder), payload_(std::move(payload)) {} + +LegacyEncodedAudioFrame::~LegacyEncodedAudioFrame() = default; + +size_t LegacyEncodedAudioFrame::Duration() const { + const int ret = decoder_->PacketDuration(payload_.data(), payload_.size()); + return (ret < 0) ? 0 : static_cast(ret); +} + +absl::optional +LegacyEncodedAudioFrame::Decode(rtc::ArrayView decoded) const { + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + const int ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + + if (ret < 0) + return absl::nullopt; + + return DecodeResult{static_cast(ret), speech_type}; +} + +std::vector LegacyEncodedAudioFrame::SplitBySamples( + AudioDecoder* decoder, + rtc::Buffer&& payload, + uint32_t timestamp, + size_t bytes_per_ms, + uint32_t timestamps_per_ms) { + RTC_DCHECK(payload.data()); + std::vector results; + size_t split_size_bytes = payload.size(); + + // Find a "chunk size" >= 20 ms and < 40 ms. + const size_t min_chunk_size = bytes_per_ms * 20; + if (min_chunk_size >= payload.size()) { + std::unique_ptr frame( + new LegacyEncodedAudioFrame(decoder, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); + } else { + // Reduce the split size by half as long as `split_size_bytes` is at least + // twice the minimum chunk size (so that the resulting size is at least as + // large as the minimum chunk size). + while (split_size_bytes >= 2 * min_chunk_size) { + split_size_bytes /= 2; + } + + const uint32_t timestamps_per_chunk = static_cast( + split_size_bytes * timestamps_per_ms / bytes_per_ms); + size_t byte_offset; + uint32_t timestamp_offset; + for (byte_offset = 0, timestamp_offset = 0; byte_offset < payload.size(); + byte_offset += split_size_bytes, + timestamp_offset += timestamps_per_chunk) { + split_size_bytes = + std::min(split_size_bytes, payload.size() - byte_offset); + rtc::Buffer new_payload(payload.data() + byte_offset, split_size_bytes); + std::unique_ptr frame( + new LegacyEncodedAudioFrame(decoder, std::move(new_payload))); + results.emplace_back(timestamp + timestamp_offset, 0, std::move(frame)); + } + } + + return results; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h b/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h new file mode 100644 index 0000000000..21da1367ed --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ +#define MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ + +#include +#include + +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/buffer.h" + +namespace webrtc { + +class LegacyEncodedAudioFrame final : public AudioDecoder::EncodedAudioFrame { + public: + LegacyEncodedAudioFrame(AudioDecoder* decoder, rtc::Buffer&& payload); + ~LegacyEncodedAudioFrame() override; + + static std::vector SplitBySamples( + AudioDecoder* decoder, + rtc::Buffer&& payload, + uint32_t timestamp, + size_t bytes_per_ms, + uint32_t timestamps_per_ms); + + size_t Duration() const override; + + absl::optional Decode( + rtc::ArrayView decoded) const override; + + // For testing: + const rtc::Buffer& payload() const { return payload_; } + + private: + AudioDecoder* const decoder_; + const rtc::Buffer payload_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_LEGACY_ENCODED_AUDIO_FRAME_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc new file mode 100644 index 0000000000..f81aeeea80 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame_unittest.cc @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" + +namespace webrtc { + +enum class NetEqDecoder { + kDecoderPCMu, + kDecoderPCMa, + kDecoderPCMu_2ch, + kDecoderPCMa_2ch, + kDecoderPCM16B, + kDecoderPCM16Bwb, + kDecoderPCM16Bswb32kHz, + kDecoderPCM16Bswb48kHz, + kDecoderPCM16B_2ch, + kDecoderPCM16Bwb_2ch, + kDecoderPCM16Bswb32kHz_2ch, + kDecoderPCM16Bswb48kHz_2ch, + kDecoderPCM16B_5ch, + kDecoderG722, +}; + +class SplitBySamplesTest : public ::testing::TestWithParam { + protected: + virtual void SetUp() { + decoder_type_ = GetParam(); + switch (decoder_type_) { + case NetEqDecoder::kDecoderPCMu: + case NetEqDecoder::kDecoderPCMa: + bytes_per_ms_ = 8; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCMu_2ch: + case NetEqDecoder::kDecoderPCMa_2ch: + bytes_per_ms_ = 2 * 8; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderG722: + bytes_per_ms_ = 8; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16B: + bytes_per_ms_ = 16; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCM16Bwb: + bytes_per_ms_ = 32; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16Bswb32kHz: + bytes_per_ms_ = 64; + samples_per_ms_ = 32; + break; + case NetEqDecoder::kDecoderPCM16Bswb48kHz: + bytes_per_ms_ = 96; + samples_per_ms_ = 48; + break; + case NetEqDecoder::kDecoderPCM16B_2ch: + bytes_per_ms_ = 2 * 16; + samples_per_ms_ = 8; + break; + case NetEqDecoder::kDecoderPCM16Bwb_2ch: + bytes_per_ms_ = 2 * 32; + samples_per_ms_ = 16; + break; + case NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch: + bytes_per_ms_ = 2 * 64; + samples_per_ms_ = 32; + break; + case NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch: + bytes_per_ms_ = 2 * 96; + samples_per_ms_ = 48; + break; + case NetEqDecoder::kDecoderPCM16B_5ch: + bytes_per_ms_ = 5 * 16; + samples_per_ms_ = 8; + break; + default: + RTC_DCHECK_NOTREACHED(); + break; + } + } + size_t bytes_per_ms_; + int samples_per_ms_; + NetEqDecoder decoder_type_; +}; + +// Test splitting sample-based payloads. +TEST_P(SplitBySamplesTest, PayloadSizes) { + constexpr uint32_t kBaseTimestamp = 0x12345678; + struct ExpectedSplit { + size_t payload_size_ms; + size_t num_frames; + // For simplicity. We only expect up to two packets per split. + size_t frame_sizes[2]; + }; + // The payloads are expected to be split as follows: + // 10 ms -> 10 ms + // 20 ms -> 20 ms + // 30 ms -> 30 ms + // 40 ms -> 20 + 20 ms + // 50 ms -> 25 + 25 ms + // 60 ms -> 30 + 30 ms + ExpectedSplit expected_splits[] = {{10, 1, {10}}, {20, 1, {20}}, + {30, 1, {30}}, {40, 2, {20, 20}}, + {50, 2, {25, 25}}, {60, 2, {30, 30}}}; + + for (const auto& expected_split : expected_splits) { + // The payload values are set to steadily increase (modulo 256), so that the + // resulting frames can be checked and we can be reasonably certain no + // sample was missed or repeated. + const auto generate_payload = [](size_t num_bytes) { + rtc::Buffer payload(num_bytes); + uint8_t value = 0; + // Allow wrap-around of value in counter below. + for (size_t i = 0; i != payload.size(); ++i, ++value) { + payload[i] = value; + } + return payload; + }; + + const auto results = LegacyEncodedAudioFrame::SplitBySamples( + nullptr, + generate_payload(expected_split.payload_size_ms * bytes_per_ms_), + kBaseTimestamp, bytes_per_ms_, samples_per_ms_); + + EXPECT_EQ(expected_split.num_frames, results.size()); + uint32_t expected_timestamp = kBaseTimestamp; + uint8_t value = 0; + for (size_t i = 0; i != expected_split.num_frames; ++i) { + const auto& result = results[i]; + const LegacyEncodedAudioFrame* frame = + static_cast(result.frame.get()); + const size_t length_bytes = expected_split.frame_sizes[i] * bytes_per_ms_; + EXPECT_EQ(length_bytes, frame->payload().size()); + EXPECT_EQ(expected_timestamp, result.timestamp); + const rtc::Buffer& payload = frame->payload(); + // Allow wrap-around of value in counter below. + for (size_t i = 0; i != payload.size(); ++i, ++value) { + ASSERT_EQ(value, payload[i]); + } + + expected_timestamp += rtc::checked_cast( + expected_split.frame_sizes[i] * samples_per_ms_); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + LegacyEncodedAudioFrame, + SplitBySamplesTest, + ::testing::Values(NetEqDecoder::kDecoderPCMu, + NetEqDecoder::kDecoderPCMa, + NetEqDecoder::kDecoderPCMu_2ch, + NetEqDecoder::kDecoderPCMa_2ch, + NetEqDecoder::kDecoderG722, + NetEqDecoder::kDecoderPCM16B, + NetEqDecoder::kDecoderPCM16Bwb, + NetEqDecoder::kDecoderPCM16Bswb32kHz, + NetEqDecoder::kDecoderPCM16Bswb48kHz, + NetEqDecoder::kDecoderPCM16B_2ch, + NetEqDecoder::kDecoderPCM16Bwb_2ch, + NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch, + NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch, + NetEqDecoder::kDecoderPCM16B_5ch)); + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/DEPS b/third_party/libwebrtc/modules/audio_coding/codecs/opus/DEPS new file mode 100644 index 0000000000..c2530726ad --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/DEPS @@ -0,0 +1,5 @@ +specific_include_rules = { + "opus_inst\.h": [ + "+third_party/opus", + ], +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.cc new file mode 100644 index 0000000000..03c02186d0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h" + +#include "absl/strings/string_view.h" + +namespace webrtc { + +absl::optional GetFormatParameter(const SdpAudioFormat& format, + absl::string_view param) { + auto it = format.parameters.find(std::string(param)); + if (it == format.parameters.end()) + return absl::nullopt; + + return it->second; +} + +// Parses a comma-separated string "1,2,0,6" into a std::vector. +template <> +absl::optional> GetFormatParameter( + const SdpAudioFormat& format, + absl::string_view param) { + std::vector result; + const std::string comma_separated_list = + GetFormatParameter(format, param).value_or(""); + size_t pos = 0; + while (pos < comma_separated_list.size()) { + const size_t next_comma = comma_separated_list.find(',', pos); + const size_t distance_to_next_comma = next_comma == std::string::npos + ? std::string::npos + : (next_comma - pos); + auto substring_with_number = + comma_separated_list.substr(pos, distance_to_next_comma); + auto conv = rtc::StringToNumber(substring_with_number); + if (!conv.has_value()) { + return absl::nullopt; + } + result.push_back(*conv); + pos += substring_with_number.size() + 1; + } + return result; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.h new file mode 100644 index 0000000000..5ebb51b577 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_coder_opus_common.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_CODER_OPUS_COMMON_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_CODER_OPUS_COMMON_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_decoder.h" +#include "api/audio_codecs/audio_format.h" +#include "rtc_base/string_to_number.h" + +namespace webrtc { + +absl::optional GetFormatParameter(const SdpAudioFormat& format, + absl::string_view param); + +template +absl::optional GetFormatParameter(const SdpAudioFormat& format, + absl::string_view param) { + return rtc::StringToNumber(GetFormatParameter(format, param).value_or("")); +} + +template <> +absl::optional> GetFormatParameter( + const SdpAudioFormat& format, + absl::string_view param); + +class OpusFrame : public AudioDecoder::EncodedAudioFrame { + public: + OpusFrame(AudioDecoder* decoder, + rtc::Buffer&& payload, + bool is_primary_payload) + : decoder_(decoder), + payload_(std::move(payload)), + is_primary_payload_(is_primary_payload) {} + + size_t Duration() const override { + int ret; + if (is_primary_payload_) { + ret = decoder_->PacketDuration(payload_.data(), payload_.size()); + } else { + ret = decoder_->PacketDurationRedundant(payload_.data(), payload_.size()); + } + return (ret < 0) ? 0 : static_cast(ret); + } + + bool IsDtxPacket() const override { return payload_.size() <= 2; } + + absl::optional Decode( + rtc::ArrayView decoded) const override { + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + int ret; + if (is_primary_payload_) { + ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } else { + ret = decoder_->DecodeRedundant( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + } + + if (ret < 0) + return absl::nullopt; + + return DecodeResult{static_cast(ret), speech_type}; + } + + private: + AudioDecoder* const decoder_; + const rtc::Buffer payload_; + const bool is_primary_payload_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_CODER_OPUS_COMMON_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.cc new file mode 100644 index 0000000000..285ea89959 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.cc @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.h" + +#include +#include +#include +#include +#include + +#include "absl/memory/memory.h" +#include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h" +#include "rtc_base/string_to_number.h" + +namespace webrtc { + +std::unique_ptr +AudioDecoderMultiChannelOpusImpl::MakeAudioDecoder( + AudioDecoderMultiChannelOpusConfig config) { + if (!config.IsOk()) { + RTC_DCHECK_NOTREACHED(); + return nullptr; + } + // Fill the pointer with a working decoder through the C interface. This + // allocates memory. + OpusDecInst* dec_state = nullptr; + const int error = WebRtcOpus_MultistreamDecoderCreate( + &dec_state, config.num_channels, config.num_streams, + config.coupled_streams, config.channel_mapping.data()); + if (error != 0) { + return nullptr; + } + + // Pass the ownership to DecoderImpl. Not using 'make_unique' because the + // c-tor is private. + return std::unique_ptr( + new AudioDecoderMultiChannelOpusImpl(dec_state, config)); +} + +AudioDecoderMultiChannelOpusImpl::AudioDecoderMultiChannelOpusImpl( + OpusDecInst* dec_state, + AudioDecoderMultiChannelOpusConfig config) + : dec_state_(dec_state), config_(config) { + RTC_DCHECK(dec_state); + WebRtcOpus_DecoderInit(dec_state_); +} + +AudioDecoderMultiChannelOpusImpl::~AudioDecoderMultiChannelOpusImpl() { + WebRtcOpus_DecoderFree(dec_state_); +} + +absl::optional +AudioDecoderMultiChannelOpusImpl::SdpToConfig(const SdpAudioFormat& format) { + AudioDecoderMultiChannelOpusConfig config; + config.num_channels = format.num_channels; + auto num_streams = GetFormatParameter(format, "num_streams"); + if (!num_streams.has_value()) { + return absl::nullopt; + } + config.num_streams = *num_streams; + + auto coupled_streams = GetFormatParameter(format, "coupled_streams"); + if (!coupled_streams.has_value()) { + return absl::nullopt; + } + config.coupled_streams = *coupled_streams; + + auto channel_mapping = + GetFormatParameter>(format, "channel_mapping"); + if (!channel_mapping.has_value()) { + return absl::nullopt; + } + config.channel_mapping = *channel_mapping; + if (!config.IsOk()) { + return absl::nullopt; + } + return config; +} + +std::vector +AudioDecoderMultiChannelOpusImpl::ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) { + std::vector results; + + if (PacketHasFec(payload.data(), payload.size())) { + const int duration = + PacketDurationRedundant(payload.data(), payload.size()); + RTC_DCHECK_GE(duration, 0); + rtc::Buffer payload_copy(payload.data(), payload.size()); + std::unique_ptr fec_frame( + new OpusFrame(this, std::move(payload_copy), false)); + results.emplace_back(timestamp - duration, 1, std::move(fec_frame)); + } + std::unique_ptr frame( + new OpusFrame(this, std::move(payload), true)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; +} + +int AudioDecoderMultiChannelOpusImpl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(sample_rate_hz, 48000); + int16_t temp_type = 1; // Default is speech. + int ret = + WebRtcOpus_Decode(dec_state_, encoded, encoded_len, decoded, &temp_type); + if (ret > 0) + ret *= static_cast( + config_.num_channels); // Return total number of samples. + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +int AudioDecoderMultiChannelOpusImpl::DecodeRedundantInternal( + const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + if (!PacketHasFec(encoded, encoded_len)) { + // This packet is a RED packet. + return DecodeInternal(encoded, encoded_len, sample_rate_hz, decoded, + speech_type); + } + + RTC_DCHECK_EQ(sample_rate_hz, 48000); + int16_t temp_type = 1; // Default is speech. + int ret = WebRtcOpus_DecodeFec(dec_state_, encoded, encoded_len, decoded, + &temp_type); + if (ret > 0) + ret *= static_cast( + config_.num_channels); // Return total number of samples. + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +void AudioDecoderMultiChannelOpusImpl::Reset() { + WebRtcOpus_DecoderInit(dec_state_); +} + +int AudioDecoderMultiChannelOpusImpl::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + return WebRtcOpus_DurationEst(dec_state_, encoded, encoded_len); +} + +int AudioDecoderMultiChannelOpusImpl::PacketDurationRedundant( + const uint8_t* encoded, + size_t encoded_len) const { + if (!PacketHasFec(encoded, encoded_len)) { + // This packet is a RED packet. + return PacketDuration(encoded, encoded_len); + } + + return WebRtcOpus_FecDurationEst(encoded, encoded_len, 48000); +} + +bool AudioDecoderMultiChannelOpusImpl::PacketHasFec(const uint8_t* encoded, + size_t encoded_len) const { + int fec; + fec = WebRtcOpus_PacketHasFec(encoded, encoded_len); + return (fec == 1); +} + +int AudioDecoderMultiChannelOpusImpl::SampleRateHz() const { + return 48000; +} + +size_t AudioDecoderMultiChannelOpusImpl::Channels() const { + return config_.num_channels; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.h new file mode 100644 index 0000000000..2ff47a8a53 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_MULTI_CHANNEL_OPUS_IMPL_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_MULTI_CHANNEL_OPUS_IMPL_H_ + +#include + +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "api/audio_codecs/audio_format.h" +#include "api/audio_codecs/opus/audio_decoder_multi_channel_opus_config.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "rtc_base/buffer.h" + +namespace webrtc { + +class AudioDecoderMultiChannelOpusImpl final : public AudioDecoder { + public: + static std::unique_ptr MakeAudioDecoder( + AudioDecoderMultiChannelOpusConfig config); + + ~AudioDecoderMultiChannelOpusImpl() override; + + AudioDecoderMultiChannelOpusImpl(const AudioDecoderMultiChannelOpusImpl&) = + delete; + AudioDecoderMultiChannelOpusImpl& operator=( + const AudioDecoderMultiChannelOpusImpl&) = delete; + + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + void Reset() override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int PacketDurationRedundant(const uint8_t* encoded, + size_t encoded_len) const override; + bool PacketHasFec(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + static absl::optional SdpToConfig( + const SdpAudioFormat& format); + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + int DecodeRedundantInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + AudioDecoderMultiChannelOpusImpl(OpusDecInst* dec_state, + AudioDecoderMultiChannelOpusConfig config); + + OpusDecInst* dec_state_; + const AudioDecoderMultiChannelOpusConfig config_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_MULTI_CHANNEL_OPUS_IMPL_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_unittest.cc new file mode 100644 index 0000000000..57e2107f3c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_unittest.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/opus/audio_decoder_multi_channel_opus.h" + +#include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +using ::testing::NiceMock; +using ::testing::Return; + +TEST(AudioDecoderMultiOpusTest, GetFormatParameter) { + const SdpAudioFormat sdp_format("multiopus", 48000, 4, + {{"channel_mapping", "0,1,2,3"}, + {"coupled_streams", "2"}, + {"num_streams", "2"}}); + + EXPECT_EQ(GetFormatParameter(sdp_format, "channel_mapping"), + absl::optional("0,1,2,3")); + + EXPECT_EQ(GetFormatParameter(sdp_format, "coupled_streams"), + absl::optional(2)); + + EXPECT_EQ(GetFormatParameter(sdp_format, "missing"), absl::nullopt); + + EXPECT_EQ(GetFormatParameter(sdp_format, "channel_mapping"), + absl::nullopt); +} + +TEST(AudioDecoderMultiOpusTest, InvalidChannelMappings) { + { + // Can't use channel 3 if there are only 2 channels. + const SdpAudioFormat sdp_format("multiopus", 48000, 2, + {{"channel_mapping", "3,0"}, + {"coupled_streams", "1"}, + {"num_streams", "2"}}); + const absl::optional decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format); + EXPECT_FALSE(decoder_config.has_value()); + } + { + // The mapping is too long. There are only 5 channels, but 6 elements in the + // mapping. + const SdpAudioFormat sdp_format("multiopus", 48000, 5, + {{"channel_mapping", "0,1,2,3,4,5"}, + {"coupled_streams", "0"}, + {"num_streams", "2"}}); + const absl::optional decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format); + EXPECT_FALSE(decoder_config.has_value()); + } + { + // The mapping doesn't parse correctly. + const SdpAudioFormat sdp_format( + "multiopus", 48000, 5, + {{"channel_mapping", "0,1,two,3,4"}, {"coupled_streams", "0"}}); + const absl::optional decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format); + EXPECT_FALSE(decoder_config.has_value()); + } +} + +TEST(AudioDecoderMultiOpusTest, ValidSdpToConfigProducesCorrectConfig) { + const SdpAudioFormat sdp_format("multiopus", 48000, 4, + {{"channel_mapping", "3,1,2,0"}, + {"coupled_streams", "2"}, + {"num_streams", "2"}}); + + const absl::optional decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format); + + ASSERT_TRUE(decoder_config.has_value()); + EXPECT_TRUE(decoder_config->IsOk()); + EXPECT_EQ(decoder_config->coupled_streams, 2); + EXPECT_THAT(decoder_config->channel_mapping, + ::testing::ContainerEq(std::vector({3, 1, 2, 0}))); +} + +TEST(AudioDecoderMultiOpusTest, InvalidSdpToConfigDoesNotProduceConfig) { + { + const SdpAudioFormat sdp_format("multiopus", 48000, 4, + {{"channel_mapping", "0,1,2,3"}, + {"coupled_stream", "2"}, + {"num_streams", "2"}}); + + const absl::optional decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format); + + EXPECT_FALSE(decoder_config.has_value()); + } + + { + const SdpAudioFormat sdp_format("multiopus", 48000, 4, + {{"channel_mapping", "0,1,2 3"}, + {"coupled_streams", "2"}, + {"num_streams", "2"}}); + + const absl::optional decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format); + + EXPECT_FALSE(decoder_config.has_value()); + } +} + +TEST(AudioDecoderMultiOpusTest, CodecsCanBeCreated) { + const SdpAudioFormat sdp_format("multiopus", 48000, 4, + {{"channel_mapping", "0,1,2,3"}, + {"coupled_streams", "2"}, + {"num_streams", "2"}}); + + const absl::optional decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(sdp_format); + + ASSERT_TRUE(decoder_config.has_value()); + + const std::unique_ptr opus_decoder = + AudioDecoderMultiChannelOpus::MakeAudioDecoder(*decoder_config); + + EXPECT_TRUE(opus_decoder); +} + +TEST(AudioDecoderMultiOpusTest, AdvertisedCodecsCanBeCreated) { + std::vector specs; + AudioDecoderMultiChannelOpus::AppendSupportedDecoders(&specs); + + EXPECT_FALSE(specs.empty()); + + for (const AudioCodecSpec& spec : specs) { + const absl::optional decoder_config = + AudioDecoderMultiChannelOpus::SdpToConfig(spec.format); + ASSERT_TRUE(decoder_config.has_value()); + + const std::unique_ptr opus_decoder = + AudioDecoderMultiChannelOpus::MakeAudioDecoder(*decoder_config); + + EXPECT_TRUE(opus_decoder); + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc new file mode 100644 index 0000000000..cff9685548 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/audio_decoder_opus.h" + +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +AudioDecoderOpusImpl::AudioDecoderOpusImpl(size_t num_channels, + int sample_rate_hz) + : channels_{num_channels}, sample_rate_hz_{sample_rate_hz} { + RTC_DCHECK(num_channels == 1 || num_channels == 2); + RTC_DCHECK(sample_rate_hz == 16000 || sample_rate_hz == 48000); + const int error = + WebRtcOpus_DecoderCreate(&dec_state_, channels_, sample_rate_hz_); + RTC_DCHECK(error == 0); + WebRtcOpus_DecoderInit(dec_state_); +} + +AudioDecoderOpusImpl::~AudioDecoderOpusImpl() { + WebRtcOpus_DecoderFree(dec_state_); +} + +std::vector AudioDecoderOpusImpl::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + std::vector results; + + if (PacketHasFec(payload.data(), payload.size())) { + const int duration = + PacketDurationRedundant(payload.data(), payload.size()); + RTC_DCHECK_GE(duration, 0); + rtc::Buffer payload_copy(payload.data(), payload.size()); + std::unique_ptr fec_frame( + new OpusFrame(this, std::move(payload_copy), false)); + results.emplace_back(timestamp - duration, 1, std::move(fec_frame)); + } + std::unique_ptr frame( + new OpusFrame(this, std::move(payload), true)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; +} + +int AudioDecoderOpusImpl::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(sample_rate_hz, sample_rate_hz_); + int16_t temp_type = 1; // Default is speech. + int ret = + WebRtcOpus_Decode(dec_state_, encoded, encoded_len, decoded, &temp_type); + if (ret > 0) + ret *= static_cast(channels_); // Return total number of samples. + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +int AudioDecoderOpusImpl::DecodeRedundantInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + if (!PacketHasFec(encoded, encoded_len)) { + // This packet is a RED packet. + return DecodeInternal(encoded, encoded_len, sample_rate_hz, decoded, + speech_type); + } + + RTC_DCHECK_EQ(sample_rate_hz, sample_rate_hz_); + int16_t temp_type = 1; // Default is speech. + int ret = WebRtcOpus_DecodeFec(dec_state_, encoded, encoded_len, decoded, + &temp_type); + if (ret > 0) + ret *= static_cast(channels_); // Return total number of samples. + *speech_type = ConvertSpeechType(temp_type); + return ret; +} + +void AudioDecoderOpusImpl::Reset() { + WebRtcOpus_DecoderInit(dec_state_); +} + +int AudioDecoderOpusImpl::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + return WebRtcOpus_DurationEst(dec_state_, encoded, encoded_len); +} + +int AudioDecoderOpusImpl::PacketDurationRedundant(const uint8_t* encoded, + size_t encoded_len) const { + if (!PacketHasFec(encoded, encoded_len)) { + // This packet is a RED packet. + return PacketDuration(encoded, encoded_len); + } + + return WebRtcOpus_FecDurationEst(encoded, encoded_len, sample_rate_hz_); +} + +bool AudioDecoderOpusImpl::PacketHasFec(const uint8_t* encoded, + size_t encoded_len) const { + int fec; + fec = WebRtcOpus_PacketHasFec(encoded, encoded_len); + return (fec == 1); +} + +int AudioDecoderOpusImpl::SampleRateHz() const { + return sample_rate_hz_; +} + +size_t AudioDecoderOpusImpl::Channels() const { + return channels_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h new file mode 100644 index 0000000000..e8fd0440bc --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_OPUS_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_OPUS_H_ + +#include +#include + +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "rtc_base/buffer.h" + +namespace webrtc { + +class AudioDecoderOpusImpl final : public AudioDecoder { + public: + explicit AudioDecoderOpusImpl(size_t num_channels, + int sample_rate_hz = 48000); + ~AudioDecoderOpusImpl() override; + + AudioDecoderOpusImpl(const AudioDecoderOpusImpl&) = delete; + AudioDecoderOpusImpl& operator=(const AudioDecoderOpusImpl&) = delete; + + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + void Reset() override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int PacketDurationRedundant(const uint8_t* encoded, + size_t encoded_len) const override; + bool PacketHasFec(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + int DecodeRedundantInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + OpusDecInst* dec_state_; + const size_t channels_; + const int sample_rate_hz_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_DECODER_OPUS_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.cc new file mode 100644 index 0000000000..38a11c123d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.cc @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * LEFT TO DO: + * - WRITE TESTS for the stuff in this file. + * - Check the creation, maybe make it safer by returning an empty optional or + * unique_ptr. --- It looks OK, but RecreateEncoderInstance can perhaps crash + * on a valid config. Can run it in the fuzzer for some time. Should prbl also + * fuzz the config. + */ + +#include "modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h" + +#include +#include +#include +#include + +#include "absl/strings/match.h" +#include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/string_to_number.h" + +namespace webrtc { + +namespace { + +// Recommended bitrates for one channel: +// 8-12 kb/s for NB speech, +// 16-20 kb/s for WB speech, +// 28-40 kb/s for FB speech, +// 48-64 kb/s for FB mono music, and +// 64-128 kb/s for FB stereo music. +// The current implementation multiplies these values by the number of channels. +constexpr int kOpusBitrateNbBps = 12000; +constexpr int kOpusBitrateWbBps = 20000; +constexpr int kOpusBitrateFbBps = 32000; + +constexpr int kDefaultMaxPlaybackRate = 48000; +// These two lists must be sorted from low to high +#if WEBRTC_OPUS_SUPPORT_120MS_PTIME +constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120}; +#else +constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60}; +#endif + +int GetBitrateBps(const AudioEncoderMultiChannelOpusConfig& config) { + RTC_DCHECK(config.IsOk()); + return config.bitrate_bps; +} +int GetMaxPlaybackRate(const SdpAudioFormat& format) { + const auto param = GetFormatParameter(format, "maxplaybackrate"); + if (param && *param >= 8000) { + return std::min(*param, kDefaultMaxPlaybackRate); + } + return kDefaultMaxPlaybackRate; +} + +int GetFrameSizeMs(const SdpAudioFormat& format) { + const auto ptime = GetFormatParameter(format, "ptime"); + if (ptime.has_value()) { + // Pick the next highest supported frame length from + // kOpusSupportedFrameLengths. + for (const int supported_frame_length : kOpusSupportedFrameLengths) { + if (supported_frame_length >= *ptime) { + return supported_frame_length; + } + } + // If none was found, return the largest supported frame length. + return *(std::end(kOpusSupportedFrameLengths) - 1); + } + + return AudioEncoderOpusConfig::kDefaultFrameSizeMs; +} + +int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) { + const int bitrate = [&] { + if (max_playback_rate <= 8000) { + return kOpusBitrateNbBps * rtc::dchecked_cast(num_channels); + } else if (max_playback_rate <= 16000) { + return kOpusBitrateWbBps * rtc::dchecked_cast(num_channels); + } else { + return kOpusBitrateFbBps * rtc::dchecked_cast(num_channels); + } + }(); + RTC_DCHECK_GE(bitrate, AudioEncoderMultiChannelOpusConfig::kMinBitrateBps); + return bitrate; +} + +// Get the maxaveragebitrate parameter in string-form, so we can properly figure +// out how invalid it is and accurately log invalid values. +int CalculateBitrate(int max_playback_rate_hz, + size_t num_channels, + absl::optional bitrate_param) { + const int default_bitrate = + CalculateDefaultBitrate(max_playback_rate_hz, num_channels); + + if (bitrate_param) { + const auto bitrate = rtc::StringToNumber(*bitrate_param); + if (bitrate) { + const int chosen_bitrate = + std::max(AudioEncoderOpusConfig::kMinBitrateBps, + std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps)); + if (bitrate != chosen_bitrate) { + RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate + << " clamped to " << chosen_bitrate; + } + return chosen_bitrate; + } + RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param + << "\" replaced by default bitrate " << default_bitrate; + } + + return default_bitrate; +} + +} // namespace + +std::unique_ptr +AudioEncoderMultiChannelOpusImpl::MakeAudioEncoder( + const AudioEncoderMultiChannelOpusConfig& config, + int payload_type) { + if (!config.IsOk()) { + RTC_DCHECK_NOTREACHED(); + return nullptr; + } + return std::make_unique(config, + payload_type); +} + +AudioEncoderMultiChannelOpusImpl::AudioEncoderMultiChannelOpusImpl( + const AudioEncoderMultiChannelOpusConfig& config, + int payload_type) + : payload_type_(payload_type), inst_(nullptr) { + RTC_DCHECK(0 <= payload_type && payload_type <= 127); + + RTC_CHECK(RecreateEncoderInstance(config)); +} + +AudioEncoderMultiChannelOpusImpl::~AudioEncoderMultiChannelOpusImpl() { + RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_)); +} + +size_t AudioEncoderMultiChannelOpusImpl::SufficientOutputBufferSize() const { + // Calculate the number of bytes we expect the encoder to produce, + // then multiply by two to give a wide margin for error. + const size_t bytes_per_millisecond = + static_cast(GetBitrateBps(config_) / (1000 * 8) + 1); + const size_t approx_encoded_bytes = + Num10msFramesPerPacket() * 10 * bytes_per_millisecond; + return 2 * approx_encoded_bytes; +} + +void AudioEncoderMultiChannelOpusImpl::Reset() { + RTC_CHECK(RecreateEncoderInstance(config_)); +} + +absl::optional> +AudioEncoderMultiChannelOpusImpl::GetFrameLengthRange() const { + return {{TimeDelta::Millis(config_.frame_size_ms), + TimeDelta::Millis(config_.frame_size_ms)}}; +} + +// If the given config is OK, recreate the Opus encoder instance with those +// settings, save the config, and return true. Otherwise, do nothing and return +// false. +bool AudioEncoderMultiChannelOpusImpl::RecreateEncoderInstance( + const AudioEncoderMultiChannelOpusConfig& config) { + if (!config.IsOk()) + return false; + config_ = config; + if (inst_) + RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_)); + input_buffer_.clear(); + input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame()); + RTC_CHECK_EQ( + 0, WebRtcOpus_MultistreamEncoderCreate( + &inst_, config.num_channels, + config.application == + AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip + ? 0 + : 1, + config.num_streams, config.coupled_streams, + config.channel_mapping.data())); + const int bitrate = GetBitrateBps(config); + RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, bitrate)); + RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps."; + if (config.fec_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_)); + RTC_LOG(LS_VERBOSE) << "Opus enable FEC"; + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_)); + RTC_LOG(LS_VERBOSE) << "Opus disable FEC"; + } + RTC_CHECK_EQ( + 0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz)); + RTC_LOG(LS_VERBOSE) << "Set Opus playback rate to " + << config.max_playback_rate_hz << " hz."; + + // Use the DEFAULT complexity. + RTC_CHECK_EQ( + 0, WebRtcOpus_SetComplexity(inst_, AudioEncoderOpusConfig().complexity)); + RTC_LOG(LS_VERBOSE) << "Set Opus coding complexity to " + << AudioEncoderOpusConfig().complexity; + + if (config.dtx_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_)); + RTC_LOG(LS_VERBOSE) << "Opus enable DTX"; + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_)); + RTC_LOG(LS_VERBOSE) << "Opus disable DTX"; + } + + if (config.cbr_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_)); + RTC_LOG(LS_VERBOSE) << "Opus enable CBR"; + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_)); + RTC_LOG(LS_VERBOSE) << "Opus disable CBR"; + } + num_channels_to_encode_ = NumChannels(); + next_frame_length_ms_ = config_.frame_size_ms; + RTC_LOG(LS_VERBOSE) << "Set Opus frame length to " << config_.frame_size_ms + << " ms"; + return true; +} + +absl::optional +AudioEncoderMultiChannelOpusImpl::SdpToConfig(const SdpAudioFormat& format) { + if (!absl::EqualsIgnoreCase(format.name, "multiopus") || + format.clockrate_hz != 48000) { + return absl::nullopt; + } + + AudioEncoderMultiChannelOpusConfig config; + config.num_channels = format.num_channels; + config.frame_size_ms = GetFrameSizeMs(format); + config.max_playback_rate_hz = GetMaxPlaybackRate(format); + config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1"); + config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1"); + config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1"); + config.bitrate_bps = + CalculateBitrate(config.max_playback_rate_hz, config.num_channels, + GetFormatParameter(format, "maxaveragebitrate")); + config.application = + config.num_channels == 1 + ? AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip + : AudioEncoderMultiChannelOpusConfig::ApplicationMode::kAudio; + + config.supported_frame_lengths_ms.clear(); + std::copy(std::begin(kOpusSupportedFrameLengths), + std::end(kOpusSupportedFrameLengths), + std::back_inserter(config.supported_frame_lengths_ms)); + + auto num_streams = GetFormatParameter(format, "num_streams"); + if (!num_streams.has_value()) { + return absl::nullopt; + } + config.num_streams = *num_streams; + + auto coupled_streams = GetFormatParameter(format, "coupled_streams"); + if (!coupled_streams.has_value()) { + return absl::nullopt; + } + config.coupled_streams = *coupled_streams; + + auto channel_mapping = + GetFormatParameter>(format, "channel_mapping"); + if (!channel_mapping.has_value()) { + return absl::nullopt; + } + config.channel_mapping = *channel_mapping; + + if (!config.IsOk()) { + return absl::nullopt; + } + return config; +} + +AudioCodecInfo AudioEncoderMultiChannelOpusImpl::QueryAudioEncoder( + const AudioEncoderMultiChannelOpusConfig& config) { + RTC_DCHECK(config.IsOk()); + AudioCodecInfo info(48000, config.num_channels, config.bitrate_bps, + AudioEncoderOpusConfig::kMinBitrateBps, + AudioEncoderOpusConfig::kMaxBitrateBps); + info.allow_comfort_noise = false; + info.supports_network_adaption = false; + return info; +} + +size_t AudioEncoderMultiChannelOpusImpl::Num10msFramesPerPacket() const { + return static_cast(rtc::CheckedDivExact(config_.frame_size_ms, 10)); +} +size_t AudioEncoderMultiChannelOpusImpl::SamplesPer10msFrame() const { + return rtc::CheckedDivExact(48000, 100) * config_.num_channels; +} +int AudioEncoderMultiChannelOpusImpl::SampleRateHz() const { + return 48000; +} +size_t AudioEncoderMultiChannelOpusImpl::NumChannels() const { + return config_.num_channels; +} +size_t AudioEncoderMultiChannelOpusImpl::Num10MsFramesInNextPacket() const { + return Num10msFramesPerPacket(); +} +size_t AudioEncoderMultiChannelOpusImpl::Max10MsFramesInAPacket() const { + return Num10msFramesPerPacket(); +} +int AudioEncoderMultiChannelOpusImpl::GetTargetBitrate() const { + return GetBitrateBps(config_); +} + +AudioEncoder::EncodedInfo AudioEncoderMultiChannelOpusImpl::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + if (input_buffer_.empty()) + first_timestamp_in_buffer_ = rtp_timestamp; + + input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend()); + if (input_buffer_.size() < + (Num10msFramesPerPacket() * SamplesPer10msFrame())) { + return EncodedInfo(); + } + RTC_CHECK_EQ(input_buffer_.size(), + Num10msFramesPerPacket() * SamplesPer10msFrame()); + + const size_t max_encoded_bytes = SufficientOutputBufferSize(); + EncodedInfo info; + info.encoded_bytes = encoded->AppendData( + max_encoded_bytes, [&](rtc::ArrayView encoded) { + int status = WebRtcOpus_Encode( + inst_, &input_buffer_[0], + rtc::CheckedDivExact(input_buffer_.size(), config_.num_channels), + rtc::saturated_cast(max_encoded_bytes), encoded.data()); + + RTC_CHECK_GE(status, 0); // Fails only if fed invalid data. + + return static_cast(status); + }); + input_buffer_.clear(); + + // Will use new packet size for next encoding. + config_.frame_size_ms = next_frame_length_ms_; + + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.send_even_if_empty = true; // Allows Opus to send empty packets. + + info.speech = true; + info.encoder_type = CodecType::kOther; + + return info; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h new file mode 100644 index 0000000000..8a7210515c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_IMPL_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_IMPL_H_ + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/audio_format.h" +#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h" +#include "api/units/time_delta.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" + +namespace webrtc { + +class RtcEventLog; + +class AudioEncoderMultiChannelOpusImpl final : public AudioEncoder { + public: + AudioEncoderMultiChannelOpusImpl( + const AudioEncoderMultiChannelOpusConfig& config, + int payload_type); + ~AudioEncoderMultiChannelOpusImpl() override; + + AudioEncoderMultiChannelOpusImpl(const AudioEncoderMultiChannelOpusImpl&) = + delete; + AudioEncoderMultiChannelOpusImpl& operator=( + const AudioEncoderMultiChannelOpusImpl&) = delete; + + // Static interface for use by BuiltinAudioEncoderFactory. + static constexpr const char* GetPayloadName() { return "multiopus"; } + static absl::optional QueryAudioEncoder( + const SdpAudioFormat& format); + + int SampleRateHz() const override; + size_t NumChannels() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + + void Reset() override; + absl::optional> GetFrameLengthRange() + const override; + + protected: + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + private: + static absl::optional SdpToConfig( + const SdpAudioFormat& format); + static AudioCodecInfo QueryAudioEncoder( + const AudioEncoderMultiChannelOpusConfig& config); + static std::unique_ptr MakeAudioEncoder( + const AudioEncoderMultiChannelOpusConfig&, + int payload_type); + + size_t Num10msFramesPerPacket() const; + size_t SamplesPer10msFrame() const; + size_t SufficientOutputBufferSize() const; + bool RecreateEncoderInstance( + const AudioEncoderMultiChannelOpusConfig& config); + void SetFrameLength(int frame_length_ms); + void SetNumChannelsToEncode(size_t num_channels_to_encode); + void SetProjectedPacketLossRate(float fraction); + + AudioEncoderMultiChannelOpusConfig config_; + const int payload_type_; + std::vector input_buffer_; + OpusEncInst* inst_; + uint32_t first_timestamp_in_buffer_; + size_t num_channels_to_encode_; + int next_frame_length_ms_; + + friend struct AudioEncoderMultiChannelOpus; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_IMPL_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_unittest.cc new file mode 100644 index 0000000000..92f6f2c169 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_unittest.cc @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus.h" + +#include "test/gmock.h" + +namespace webrtc { +using ::testing::NiceMock; +using ::testing::Return; + +namespace { +constexpr int kOpusPayloadType = 120; +} // namespace + +TEST(AudioEncoderMultiOpusTest, CheckConfigValidity) { + { + const SdpAudioFormat sdp_format("multiopus", 48000, 2, + {{"channel_mapping", "3,0"}, + {"coupled_streams", "1"}, + {"num_streams", "2"}}); + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + + // Maps input channel 0 to coded channel 3, which doesn't exist. + EXPECT_FALSE(encoder_config.has_value()); + } + + { + const SdpAudioFormat sdp_format("multiopus", 48000, 2, + {{"channel_mapping", "0"}, + {"coupled_streams", "1"}, + {"num_streams", "2"}}); + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + + // The mapping is too short. + EXPECT_FALSE(encoder_config.has_value()); + } + { + const SdpAudioFormat sdp_format("multiopus", 48000, 3, + {{"channel_mapping", "0,0,0"}, + {"coupled_streams", "0"}, + {"num_streams", "1"}}); + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + + // Coded channel 0 comes from both input channels 0, 1 and 2. + EXPECT_FALSE(encoder_config.has_value()); + } + { + const SdpAudioFormat sdp_format("multiopus", 48000, 3, + {{"channel_mapping", "0,255,255"}, + {"coupled_streams", "0"}, + {"num_streams", "1"}}); + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + ASSERT_TRUE(encoder_config.has_value()); + + // This is fine, because channels 1, 2 are set to be ignored. + EXPECT_TRUE(encoder_config->IsOk()); + } + { + const SdpAudioFormat sdp_format("multiopus", 48000, 3, + {{"channel_mapping", "0,255,255"}, + {"coupled_streams", "0"}, + {"num_streams", "2"}}); + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + + // This is NOT fine, because channels nothing says how coded channel 1 + // should be coded. + EXPECT_FALSE(encoder_config.has_value()); + } +} + +TEST(AudioEncoderMultiOpusTest, ConfigValuesAreParsedCorrectly) { + SdpAudioFormat sdp_format({"multiopus", + 48000, + 6, + {{"minptime", "10"}, + {"useinbandfec", "1"}, + {"channel_mapping", "0,4,1,2,3,5"}, + {"num_streams", "4"}, + {"coupled_streams", "2"}}}); + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + ASSERT_TRUE(encoder_config.has_value()); + + EXPECT_EQ(encoder_config->coupled_streams, 2); + EXPECT_EQ(encoder_config->num_streams, 4); + EXPECT_THAT( + encoder_config->channel_mapping, + testing::ContainerEq(std::vector({0, 4, 1, 2, 3, 5}))); +} + +TEST(AudioEncoderMultiOpusTest, CreateFromValidConfig) { + { + const SdpAudioFormat sdp_format("multiopus", 48000, 3, + {{"channel_mapping", "0,255,255"}, + {"coupled_streams", "0"}, + {"num_streams", "2"}}); + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + ASSERT_FALSE(encoder_config.has_value()); + } + { + const SdpAudioFormat sdp_format("multiopus", 48000, 3, + {{"channel_mapping", "1,255,0"}, + {"coupled_streams", "1"}, + {"num_streams", "1"}}); + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(sdp_format); + ASSERT_TRUE(encoder_config.has_value()); + + EXPECT_THAT(encoder_config->channel_mapping, + testing::ContainerEq(std::vector({1, 255, 0}))); + + EXPECT_TRUE(encoder_config->IsOk()); + + const std::unique_ptr opus_encoder = + AudioEncoderMultiChannelOpus::MakeAudioEncoder(*encoder_config, + kOpusPayloadType); + + // Creating an encoder from a valid config should work. + EXPECT_TRUE(opus_encoder); + } +} + +TEST(AudioEncoderMultiOpusTest, AdvertisedCodecsCanBeCreated) { + std::vector specs; + AudioEncoderMultiChannelOpus::AppendSupportedEncoders(&specs); + + EXPECT_FALSE(specs.empty()); + + for (const AudioCodecSpec& spec : specs) { + const absl::optional encoder_config = + AudioEncoderMultiChannelOpus::SdpToConfig(spec.format); + ASSERT_TRUE(encoder_config.has_value()); + + const std::unique_ptr opus_encoder = + AudioEncoderMultiChannelOpus::MakeAudioEncoder(*encoder_config, + kOpusPayloadType); + + EXPECT_TRUE(opus_encoder); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc new file mode 100644 index 0000000000..17e0e33b1d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -0,0 +1,824 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/audio_encoder_opus.h" + +#include +#include +#include +#include +#include + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" +#include "modules/audio_coding/audio_network_adaptor/controller_manager.h" +#include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/string_encode.h" +#include "rtc_base/string_to_number.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { + +// Codec parameters for Opus. +// draft-spittka-payload-rtp-opus-03 + +// Recommended bitrates: +// 8-12 kb/s for NB speech, +// 16-20 kb/s for WB speech, +// 28-40 kb/s for FB speech, +// 48-64 kb/s for FB mono music, and +// 64-128 kb/s for FB stereo music. +// The current implementation applies the following values to mono signals, +// and multiplies them by 2 for stereo. +constexpr int kOpusBitrateNbBps = 12000; +constexpr int kOpusBitrateWbBps = 20000; +constexpr int kOpusBitrateFbBps = 32000; + +constexpr int kRtpTimestampRateHz = 48000; +constexpr int kDefaultMaxPlaybackRate = 48000; + +// These two lists must be sorted from low to high +#if WEBRTC_OPUS_SUPPORT_120MS_PTIME +constexpr int kANASupportedFrameLengths[] = {20, 40, 60, 120}; +constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120}; +#else +constexpr int kANASupportedFrameLengths[] = {20, 40, 60}; +constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60}; +#endif + +// PacketLossFractionSmoother uses an exponential filter with a time constant +// of -1.0 / ln(0.9999) = 10000 ms. +constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f; +constexpr float kMaxPacketLossFraction = 0.2f; + +int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) { + const int bitrate = [&] { + if (max_playback_rate <= 8000) { + return kOpusBitrateNbBps * rtc::dchecked_cast(num_channels); + } else if (max_playback_rate <= 16000) { + return kOpusBitrateWbBps * rtc::dchecked_cast(num_channels); + } else { + return kOpusBitrateFbBps * rtc::dchecked_cast(num_channels); + } + }(); + RTC_DCHECK_GE(bitrate, AudioEncoderOpusConfig::kMinBitrateBps); + RTC_DCHECK_LE(bitrate, AudioEncoderOpusConfig::kMaxBitrateBps); + return bitrate; +} + +// Get the maxaveragebitrate parameter in string-form, so we can properly figure +// out how invalid it is and accurately log invalid values. +int CalculateBitrate(int max_playback_rate_hz, + size_t num_channels, + absl::optional bitrate_param) { + const int default_bitrate = + CalculateDefaultBitrate(max_playback_rate_hz, num_channels); + + if (bitrate_param) { + const auto bitrate = rtc::StringToNumber(*bitrate_param); + if (bitrate) { + const int chosen_bitrate = + std::max(AudioEncoderOpusConfig::kMinBitrateBps, + std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps)); + if (bitrate != chosen_bitrate) { + RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate + << " clamped to " << chosen_bitrate; + } + return chosen_bitrate; + } + RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param + << "\" replaced by default bitrate " << default_bitrate; + } + + return default_bitrate; +} + +int GetChannelCount(const SdpAudioFormat& format) { + const auto param = GetFormatParameter(format, "stereo"); + if (param == "1") { + return 2; + } else { + return 1; + } +} + +int GetMaxPlaybackRate(const SdpAudioFormat& format) { + const auto param = GetFormatParameter(format, "maxplaybackrate"); + if (param && *param >= 8000) { + return std::min(*param, kDefaultMaxPlaybackRate); + } + return kDefaultMaxPlaybackRate; +} + +int GetFrameSizeMs(const SdpAudioFormat& format) { + const auto ptime = GetFormatParameter(format, "ptime"); + if (ptime) { + // Pick the next highest supported frame length from + // kOpusSupportedFrameLengths. + for (const int supported_frame_length : kOpusSupportedFrameLengths) { + if (supported_frame_length >= *ptime) { + return supported_frame_length; + } + } + // If none was found, return the largest supported frame length. + return *(std::end(kOpusSupportedFrameLengths) - 1); + } + + return AudioEncoderOpusConfig::kDefaultFrameSizeMs; +} + +void FindSupportedFrameLengths(int min_frame_length_ms, + int max_frame_length_ms, + std::vector* out) { + out->clear(); + std::copy_if(std::begin(kANASupportedFrameLengths), + std::end(kANASupportedFrameLengths), std::back_inserter(*out), + [&](int frame_length_ms) { + return frame_length_ms >= min_frame_length_ms && + frame_length_ms <= max_frame_length_ms; + }); + RTC_DCHECK(std::is_sorted(out->begin(), out->end())); +} + +int GetBitrateBps(const AudioEncoderOpusConfig& config) { + RTC_DCHECK(config.IsOk()); + return *config.bitrate_bps; +} + +std::vector GetBitrateMultipliers() { + constexpr char kBitrateMultipliersName[] = + "WebRTC-Audio-OpusBitrateMultipliers"; + const bool use_bitrate_multipliers = + webrtc::field_trial::IsEnabled(kBitrateMultipliersName); + if (use_bitrate_multipliers) { + const std::string field_trial_string = + webrtc::field_trial::FindFullName(kBitrateMultipliersName); + std::vector pieces; + rtc::tokenize(field_trial_string, '-', &pieces); + if (pieces.size() < 2 || pieces[0] != "Enabled") { + RTC_LOG(LS_WARNING) << "Invalid parameters for " + << kBitrateMultipliersName + << ", not using custom values."; + return std::vector(); + } + std::vector multipliers(pieces.size() - 1); + for (size_t i = 1; i < pieces.size(); i++) { + if (!rtc::FromString(pieces[i], &multipliers[i - 1])) { + RTC_LOG(LS_WARNING) + << "Invalid parameters for " << kBitrateMultipliersName + << ", not using custom values."; + return std::vector(); + } + } + RTC_LOG(LS_INFO) << "Using custom bitrate multipliers: " + << field_trial_string; + return multipliers; + } + return std::vector(); +} + +int GetMultipliedBitrate(int bitrate, const std::vector& multipliers) { + // The multipliers are valid from 5 kbps. + const size_t bitrate_kbps = static_cast(bitrate / 1000); + if (bitrate_kbps < 5 || bitrate_kbps >= multipliers.size() + 5) { + return bitrate; + } + return static_cast(multipliers[bitrate_kbps - 5] * bitrate); +} +} // namespace + +void AudioEncoderOpusImpl::AppendSupportedEncoders( + std::vector* specs) { + const SdpAudioFormat fmt = {"opus", + kRtpTimestampRateHz, + 2, + {{"minptime", "10"}, {"useinbandfec", "1"}}}; + const AudioCodecInfo info = QueryAudioEncoder(*SdpToConfig(fmt)); + specs->push_back({fmt, info}); +} + +AudioCodecInfo AudioEncoderOpusImpl::QueryAudioEncoder( + const AudioEncoderOpusConfig& config) { + RTC_DCHECK(config.IsOk()); + AudioCodecInfo info(config.sample_rate_hz, config.num_channels, + *config.bitrate_bps, + AudioEncoderOpusConfig::kMinBitrateBps, + AudioEncoderOpusConfig::kMaxBitrateBps); + info.allow_comfort_noise = false; + info.supports_network_adaption = true; + return info; +} + +std::unique_ptr AudioEncoderOpusImpl::MakeAudioEncoder( + const AudioEncoderOpusConfig& config, + int payload_type) { + if (!config.IsOk()) { + RTC_DCHECK_NOTREACHED(); + return nullptr; + } + return std::make_unique(config, payload_type); +} + +absl::optional AudioEncoderOpusImpl::SdpToConfig( + const SdpAudioFormat& format) { + if (!absl::EqualsIgnoreCase(format.name, "opus") || + format.clockrate_hz != kRtpTimestampRateHz) { + return absl::nullopt; + } + + AudioEncoderOpusConfig config; + config.num_channels = GetChannelCount(format); + config.frame_size_ms = GetFrameSizeMs(format); + config.max_playback_rate_hz = GetMaxPlaybackRate(format); + config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1"); + config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1"); + config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1"); + config.bitrate_bps = + CalculateBitrate(config.max_playback_rate_hz, config.num_channels, + GetFormatParameter(format, "maxaveragebitrate")); + config.application = config.num_channels == 1 + ? AudioEncoderOpusConfig::ApplicationMode::kVoip + : AudioEncoderOpusConfig::ApplicationMode::kAudio; + + constexpr int kMinANAFrameLength = kANASupportedFrameLengths[0]; + constexpr int kMaxANAFrameLength = + kANASupportedFrameLengths[arraysize(kANASupportedFrameLengths) - 1]; + + // For now, minptime and maxptime are only used with ANA. If ptime is outside + // of this range, it will get adjusted once ANA takes hold. Ideally, we'd know + // if ANA was to be used when setting up the config, and adjust accordingly. + const int min_frame_length_ms = + GetFormatParameter(format, "minptime").value_or(kMinANAFrameLength); + const int max_frame_length_ms = + GetFormatParameter(format, "maxptime").value_or(kMaxANAFrameLength); + + FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms, + &config.supported_frame_lengths_ms); + if (!config.IsOk()) { + RTC_DCHECK_NOTREACHED(); + return absl::nullopt; + } + return config; +} + +absl::optional AudioEncoderOpusImpl::GetNewComplexity( + const AudioEncoderOpusConfig& config) { + RTC_DCHECK(config.IsOk()); + const int bitrate_bps = GetBitrateBps(config); + if (bitrate_bps >= config.complexity_threshold_bps - + config.complexity_threshold_window_bps && + bitrate_bps <= config.complexity_threshold_bps + + config.complexity_threshold_window_bps) { + // Within the hysteresis window; make no change. + return absl::nullopt; + } else { + return bitrate_bps <= config.complexity_threshold_bps + ? config.low_rate_complexity + : config.complexity; + } +} + +absl::optional AudioEncoderOpusImpl::GetNewBandwidth( + const AudioEncoderOpusConfig& config, + OpusEncInst* inst) { + constexpr int kMinWidebandBitrate = 8000; + constexpr int kMaxNarrowbandBitrate = 9000; + constexpr int kAutomaticThreshold = 11000; + RTC_DCHECK(config.IsOk()); + const int bitrate = GetBitrateBps(config); + if (bitrate > kAutomaticThreshold) { + return absl::optional(OPUS_AUTO); + } + const int bandwidth = WebRtcOpus_GetBandwidth(inst); + RTC_DCHECK_GE(bandwidth, 0); + if (bitrate > kMaxNarrowbandBitrate && bandwidth < OPUS_BANDWIDTH_WIDEBAND) { + return absl::optional(OPUS_BANDWIDTH_WIDEBAND); + } else if (bitrate < kMinWidebandBitrate && + bandwidth > OPUS_BANDWIDTH_NARROWBAND) { + return absl::optional(OPUS_BANDWIDTH_NARROWBAND); + } + return absl::optional(); +} + +class AudioEncoderOpusImpl::PacketLossFractionSmoother { + public: + explicit PacketLossFractionSmoother() + : last_sample_time_ms_(rtc::TimeMillis()), + smoother_(kAlphaForPacketLossFractionSmoother) {} + + // Gets the smoothed packet loss fraction. + float GetAverage() const { + float value = smoother_.filtered(); + return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value; + } + + // Add new observation to the packet loss fraction smoother. + void AddSample(float packet_loss_fraction) { + int64_t now_ms = rtc::TimeMillis(); + smoother_.Apply(static_cast(now_ms - last_sample_time_ms_), + packet_loss_fraction); + last_sample_time_ms_ = now_ms; + } + + private: + int64_t last_sample_time_ms_; + + // An exponential filter is used to smooth the packet loss fraction. + rtc::ExpFilter smoother_; +}; + +AudioEncoderOpusImpl::AudioEncoderOpusImpl(const AudioEncoderOpusConfig& config, + int payload_type) + : AudioEncoderOpusImpl( + config, + payload_type, + [this](absl::string_view config_string, RtcEventLog* event_log) { + return DefaultAudioNetworkAdaptorCreator(config_string, event_log); + }, + // We choose 5sec as initial time constant due to empirical data. + std::make_unique(5000)) {} + +AudioEncoderOpusImpl::AudioEncoderOpusImpl( + const AudioEncoderOpusConfig& config, + int payload_type, + const AudioNetworkAdaptorCreator& audio_network_adaptor_creator, + std::unique_ptr bitrate_smoother) + : payload_type_(payload_type), + use_stable_target_for_adaptation_(!webrtc::field_trial::IsDisabled( + "WebRTC-Audio-StableTargetAdaptation")), + adjust_bandwidth_( + webrtc::field_trial::IsEnabled("WebRTC-AdjustOpusBandwidth")), + bitrate_changed_(true), + bitrate_multipliers_(GetBitrateMultipliers()), + packet_loss_rate_(0.0), + inst_(nullptr), + packet_loss_fraction_smoother_(new PacketLossFractionSmoother()), + audio_network_adaptor_creator_(audio_network_adaptor_creator), + bitrate_smoother_(std::move(bitrate_smoother)), + consecutive_dtx_frames_(0) { + RTC_DCHECK(0 <= payload_type && payload_type <= 127); + + // Sanity check of the redundant payload type field that we want to get rid + // of. See https://bugs.chromium.org/p/webrtc/issues/detail?id=7847 + RTC_CHECK(config.payload_type == -1 || config.payload_type == payload_type); + + RTC_CHECK(RecreateEncoderInstance(config)); + SetProjectedPacketLossRate(packet_loss_rate_); +} + +AudioEncoderOpusImpl::AudioEncoderOpusImpl(int payload_type, + const SdpAudioFormat& format) + : AudioEncoderOpusImpl(*SdpToConfig(format), payload_type) {} + +AudioEncoderOpusImpl::~AudioEncoderOpusImpl() { + RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_)); +} + +int AudioEncoderOpusImpl::SampleRateHz() const { + return config_.sample_rate_hz; +} + +size_t AudioEncoderOpusImpl::NumChannels() const { + return config_.num_channels; +} + +int AudioEncoderOpusImpl::RtpTimestampRateHz() const { + return kRtpTimestampRateHz; +} + +size_t AudioEncoderOpusImpl::Num10MsFramesInNextPacket() const { + return Num10msFramesPerPacket(); +} + +size_t AudioEncoderOpusImpl::Max10MsFramesInAPacket() const { + return Num10msFramesPerPacket(); +} + +int AudioEncoderOpusImpl::GetTargetBitrate() const { + return GetBitrateBps(config_); +} + +void AudioEncoderOpusImpl::Reset() { + RTC_CHECK(RecreateEncoderInstance(config_)); +} + +bool AudioEncoderOpusImpl::SetFec(bool enable) { + if (enable) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_)); + } + config_.fec_enabled = enable; + return true; +} + +bool AudioEncoderOpusImpl::SetDtx(bool enable) { + if (enable) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_)); + } + config_.dtx_enabled = enable; + return true; +} + +bool AudioEncoderOpusImpl::GetDtx() const { + return config_.dtx_enabled; +} + +bool AudioEncoderOpusImpl::SetApplication(Application application) { + auto conf = config_; + switch (application) { + case Application::kSpeech: + conf.application = AudioEncoderOpusConfig::ApplicationMode::kVoip; + break; + case Application::kAudio: + conf.application = AudioEncoderOpusConfig::ApplicationMode::kAudio; + break; + } + return RecreateEncoderInstance(conf); +} + +void AudioEncoderOpusImpl::SetMaxPlaybackRate(int frequency_hz) { + auto conf = config_; + conf.max_playback_rate_hz = frequency_hz; + RTC_CHECK(RecreateEncoderInstance(conf)); +} + +bool AudioEncoderOpusImpl::EnableAudioNetworkAdaptor( + const std::string& config_string, + RtcEventLog* event_log) { + audio_network_adaptor_ = + audio_network_adaptor_creator_(config_string, event_log); + return audio_network_adaptor_.get() != nullptr; +} + +void AudioEncoderOpusImpl::DisableAudioNetworkAdaptor() { + audio_network_adaptor_.reset(nullptr); +} + +void AudioEncoderOpusImpl::OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + if (audio_network_adaptor_) { + audio_network_adaptor_->SetUplinkPacketLossFraction( + uplink_packet_loss_fraction); + ApplyAudioNetworkAdaptor(); + } + packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction); + float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage(); + SetProjectedPacketLossRate(average_fraction_loss); +} + +void AudioEncoderOpusImpl::OnReceivedTargetAudioBitrate( + int target_audio_bitrate_bps) { + SetTargetBitrate(target_audio_bitrate_bps); +} + +void AudioEncoderOpusImpl::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional bwe_period_ms, + absl::optional stable_target_bitrate_bps) { + if (audio_network_adaptor_) { + audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps); + if (use_stable_target_for_adaptation_) { + if (stable_target_bitrate_bps) + audio_network_adaptor_->SetUplinkBandwidth(*stable_target_bitrate_bps); + } else { + // We give smoothed bitrate allocation to audio network adaptor as + // the uplink bandwidth. + // The BWE spikes should not affect the bitrate smoother more than 25%. + // To simplify the calculations we use a step response as input signal. + // The step response of an exponential filter is + // u(t) = 1 - e^(-t / time_constant). + // In order to limit the affect of a BWE spike within 25% of its value + // before + // the next BWE update, we would choose a time constant that fulfills + // 1 - e^(-bwe_period_ms / time_constant) < 0.25 + // Then 4 * bwe_period_ms is a good choice. + if (bwe_period_ms) + bitrate_smoother_->SetTimeConstantMs(*bwe_period_ms * 4); + bitrate_smoother_->AddSample(target_audio_bitrate_bps); + } + + ApplyAudioNetworkAdaptor(); + } else { + if (!overhead_bytes_per_packet_) { + RTC_LOG(LS_INFO) + << "AudioEncoderOpusImpl: Overhead unknown, target audio bitrate " + << target_audio_bitrate_bps << " bps is ignored."; + return; + } + const int overhead_bps = static_cast( + *overhead_bytes_per_packet_ * 8 * 100 / Num10MsFramesInNextPacket()); + SetTargetBitrate( + std::min(AudioEncoderOpusConfig::kMaxBitrateBps, + std::max(AudioEncoderOpusConfig::kMinBitrateBps, + target_audio_bitrate_bps - overhead_bps))); + } +} +void AudioEncoderOpusImpl::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional bwe_period_ms) { + OnReceivedUplinkBandwidth(target_audio_bitrate_bps, bwe_period_ms, + absl::nullopt); +} + +void AudioEncoderOpusImpl::OnReceivedUplinkAllocation( + BitrateAllocationUpdate update) { + OnReceivedUplinkBandwidth(update.target_bitrate.bps(), update.bwe_period.ms(), + update.stable_target_bitrate.bps()); +} + +void AudioEncoderOpusImpl::OnReceivedRtt(int rtt_ms) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetRtt(rtt_ms); + ApplyAudioNetworkAdaptor(); +} + +void AudioEncoderOpusImpl::OnReceivedOverhead( + size_t overhead_bytes_per_packet) { + if (audio_network_adaptor_) { + audio_network_adaptor_->SetOverhead(overhead_bytes_per_packet); + ApplyAudioNetworkAdaptor(); + } else { + overhead_bytes_per_packet_ = overhead_bytes_per_packet; + } +} + +void AudioEncoderOpusImpl::SetReceiverFrameLengthRange( + int min_frame_length_ms, + int max_frame_length_ms) { + // Ensure that `SetReceiverFrameLengthRange` is called before + // `EnableAudioNetworkAdaptor`, otherwise we need to recreate + // `audio_network_adaptor_`, which is not a needed use case. + RTC_DCHECK(!audio_network_adaptor_); + FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms, + &config_.supported_frame_lengths_ms); +} + +AudioEncoder::EncodedInfo AudioEncoderOpusImpl::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + MaybeUpdateUplinkBandwidth(); + + if (input_buffer_.empty()) + first_timestamp_in_buffer_ = rtp_timestamp; + + input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend()); + if (input_buffer_.size() < + (Num10msFramesPerPacket() * SamplesPer10msFrame())) { + return EncodedInfo(); + } + RTC_CHECK_EQ(input_buffer_.size(), + Num10msFramesPerPacket() * SamplesPer10msFrame()); + + const size_t max_encoded_bytes = SufficientOutputBufferSize(); + EncodedInfo info; + info.encoded_bytes = encoded->AppendData( + max_encoded_bytes, [&](rtc::ArrayView encoded) { + int status = WebRtcOpus_Encode( + inst_, &input_buffer_[0], + rtc::CheckedDivExact(input_buffer_.size(), config_.num_channels), + rtc::saturated_cast(max_encoded_bytes), encoded.data()); + + RTC_CHECK_GE(status, 0); // Fails only if fed invalid data. + + return static_cast(status); + }); + input_buffer_.clear(); + + bool dtx_frame = (info.encoded_bytes <= 2); + + // Will use new packet size for next encoding. + config_.frame_size_ms = next_frame_length_ms_; + + if (adjust_bandwidth_ && bitrate_changed_) { + const auto bandwidth = GetNewBandwidth(config_, inst_); + if (bandwidth) { + RTC_CHECK_EQ(0, WebRtcOpus_SetBandwidth(inst_, *bandwidth)); + } + bitrate_changed_ = false; + } + + info.encoded_timestamp = first_timestamp_in_buffer_; + info.payload_type = payload_type_; + info.send_even_if_empty = true; // Allows Opus to send empty packets. + // After 20 DTX frames (MAX_CONSECUTIVE_DTX) Opus will send a frame + // coding the background noise. Avoid flagging this frame as speech + // (even though there is a probability of the frame being speech). + info.speech = !dtx_frame && (consecutive_dtx_frames_ != 20); + info.encoder_type = CodecType::kOpus; + + // Increase or reset DTX counter. + consecutive_dtx_frames_ = (dtx_frame) ? (consecutive_dtx_frames_ + 1) : (0); + + return info; +} + +size_t AudioEncoderOpusImpl::Num10msFramesPerPacket() const { + return static_cast(rtc::CheckedDivExact(config_.frame_size_ms, 10)); +} + +size_t AudioEncoderOpusImpl::SamplesPer10msFrame() const { + return rtc::CheckedDivExact(config_.sample_rate_hz, 100) * + config_.num_channels; +} + +size_t AudioEncoderOpusImpl::SufficientOutputBufferSize() const { + // Calculate the number of bytes we expect the encoder to produce, + // then multiply by two to give a wide margin for error. + const size_t bytes_per_millisecond = + static_cast(GetBitrateBps(config_) / (1000 * 8) + 1); + const size_t approx_encoded_bytes = + Num10msFramesPerPacket() * 10 * bytes_per_millisecond; + return 2 * approx_encoded_bytes; +} + +// If the given config is OK, recreate the Opus encoder instance with those +// settings, save the config, and return true. Otherwise, do nothing and return +// false. +bool AudioEncoderOpusImpl::RecreateEncoderInstance( + const AudioEncoderOpusConfig& config) { + if (!config.IsOk()) + return false; + config_ = config; + if (inst_) + RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_)); + input_buffer_.clear(); + input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame()); + RTC_CHECK_EQ(0, WebRtcOpus_EncoderCreate( + &inst_, config.num_channels, + config.application == + AudioEncoderOpusConfig::ApplicationMode::kVoip + ? 0 + : 1, + config.sample_rate_hz)); + const int bitrate = GetBitrateBps(config); + RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, bitrate)); + RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps."; + if (config.fec_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_)); + } + RTC_CHECK_EQ( + 0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz)); + // Use the default complexity if the start bitrate is within the hysteresis + // window. + complexity_ = GetNewComplexity(config).value_or(config.complexity); + RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_)); + bitrate_changed_ = true; + if (config.dtx_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_)); + } + RTC_CHECK_EQ(0, + WebRtcOpus_SetPacketLossRate( + inst_, static_cast(packet_loss_rate_ * 100 + .5))); + if (config.cbr_enabled) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_)); + } + num_channels_to_encode_ = NumChannels(); + next_frame_length_ms_ = config_.frame_size_ms; + return true; +} + +void AudioEncoderOpusImpl::SetFrameLength(int frame_length_ms) { + if (next_frame_length_ms_ != frame_length_ms) { + RTC_LOG(LS_VERBOSE) << "Update Opus frame length " + << "from " << next_frame_length_ms_ << " ms " + << "to " << frame_length_ms << " ms."; + } + next_frame_length_ms_ = frame_length_ms; +} + +void AudioEncoderOpusImpl::SetNumChannelsToEncode( + size_t num_channels_to_encode) { + RTC_DCHECK_GT(num_channels_to_encode, 0); + RTC_DCHECK_LE(num_channels_to_encode, config_.num_channels); + + if (num_channels_to_encode_ == num_channels_to_encode) + return; + + RTC_CHECK_EQ(0, WebRtcOpus_SetForceChannels(inst_, num_channels_to_encode)); + num_channels_to_encode_ = num_channels_to_encode; +} + +void AudioEncoderOpusImpl::SetProjectedPacketLossRate(float fraction) { + fraction = std::min(std::max(fraction, 0.0f), kMaxPacketLossFraction); + if (packet_loss_rate_ != fraction) { + packet_loss_rate_ = fraction; + RTC_CHECK_EQ( + 0, WebRtcOpus_SetPacketLossRate( + inst_, static_cast(packet_loss_rate_ * 100 + .5))); + } +} + +void AudioEncoderOpusImpl::SetTargetBitrate(int bits_per_second) { + const int new_bitrate = rtc::SafeClamp( + bits_per_second, AudioEncoderOpusConfig::kMinBitrateBps, + AudioEncoderOpusConfig::kMaxBitrateBps); + if (config_.bitrate_bps && *config_.bitrate_bps != new_bitrate) { + config_.bitrate_bps = new_bitrate; + RTC_DCHECK(config_.IsOk()); + const int bitrate = GetBitrateBps(config_); + RTC_CHECK_EQ( + 0, WebRtcOpus_SetBitRate( + inst_, GetMultipliedBitrate(bitrate, bitrate_multipliers_))); + RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps."; + bitrate_changed_ = true; + } + + const auto new_complexity = GetNewComplexity(config_); + if (new_complexity && complexity_ != *new_complexity) { + complexity_ = *new_complexity; + RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_)); + } +} + +void AudioEncoderOpusImpl::ApplyAudioNetworkAdaptor() { + auto config = audio_network_adaptor_->GetEncoderRuntimeConfig(); + + if (config.bitrate_bps) + SetTargetBitrate(*config.bitrate_bps); + if (config.frame_length_ms) + SetFrameLength(*config.frame_length_ms); + if (config.enable_dtx) + SetDtx(*config.enable_dtx); + if (config.num_channels) + SetNumChannelsToEncode(*config.num_channels); +} + +std::unique_ptr +AudioEncoderOpusImpl::DefaultAudioNetworkAdaptorCreator( + absl::string_view config_string, + RtcEventLog* event_log) const { + AudioNetworkAdaptorImpl::Config config; + config.event_log = event_log; + return std::unique_ptr(new AudioNetworkAdaptorImpl( + config, ControllerManagerImpl::Create( + config_string, NumChannels(), supported_frame_lengths_ms(), + AudioEncoderOpusConfig::kMinBitrateBps, + num_channels_to_encode_, next_frame_length_ms_, + GetTargetBitrate(), config_.fec_enabled, GetDtx()))); +} + +void AudioEncoderOpusImpl::MaybeUpdateUplinkBandwidth() { + if (audio_network_adaptor_ && !use_stable_target_for_adaptation_) { + int64_t now_ms = rtc::TimeMillis(); + if (!bitrate_smoother_last_update_time_ || + now_ms - *bitrate_smoother_last_update_time_ >= + config_.uplink_bandwidth_update_interval_ms) { + absl::optional smoothed_bitrate = bitrate_smoother_->GetAverage(); + if (smoothed_bitrate) + audio_network_adaptor_->SetUplinkBandwidth(*smoothed_bitrate); + bitrate_smoother_last_update_time_ = now_ms; + } + } +} + +ANAStats AudioEncoderOpusImpl::GetANAStats() const { + if (audio_network_adaptor_) { + return audio_network_adaptor_->GetStats(); + } + return ANAStats(); +} + +absl::optional > +AudioEncoderOpusImpl::GetFrameLengthRange() const { + if (audio_network_adaptor_) { + if (config_.supported_frame_lengths_ms.empty()) { + return absl::nullopt; + } + return {{TimeDelta::Millis(config_.supported_frame_lengths_ms.front()), + TimeDelta::Millis(config_.supported_frame_lengths_ms.back())}}; + } else { + return {{TimeDelta::Millis(config_.frame_size_ms), + TimeDelta::Millis(config_.frame_size_ms)}}; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h new file mode 100644 index 0000000000..8c5c235016 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/audio_codecs/audio_format.h" +#include "api/audio_codecs/opus/audio_encoder_opus_config.h" +#include "common_audio/smoothing_filter.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" + +namespace webrtc { + +class RtcEventLog; + +class AudioEncoderOpusImpl final : public AudioEncoder { + public: + // Returns empty if the current bitrate falls within the hysteresis window, + // defined by complexity_threshold_bps +/- complexity_threshold_window_bps. + // Otherwise, returns the current complexity depending on whether the + // current bitrate is above or below complexity_threshold_bps. + static absl::optional GetNewComplexity( + const AudioEncoderOpusConfig& config); + + // Returns OPUS_AUTO if the the current bitrate is above wideband threshold. + // Returns empty if it is below, but bandwidth coincides with the desired one. + // Otherwise returns the desired bandwidth. + static absl::optional GetNewBandwidth( + const AudioEncoderOpusConfig& config, + OpusEncInst* inst); + + using AudioNetworkAdaptorCreator = + std::function(absl::string_view, + RtcEventLog*)>; + + AudioEncoderOpusImpl(const AudioEncoderOpusConfig& config, int payload_type); + + // Dependency injection for testing. + AudioEncoderOpusImpl( + const AudioEncoderOpusConfig& config, + int payload_type, + const AudioNetworkAdaptorCreator& audio_network_adaptor_creator, + std::unique_ptr bitrate_smoother); + + AudioEncoderOpusImpl(int payload_type, const SdpAudioFormat& format); + ~AudioEncoderOpusImpl() override; + + AudioEncoderOpusImpl(const AudioEncoderOpusImpl&) = delete; + AudioEncoderOpusImpl& operator=(const AudioEncoderOpusImpl&) = delete; + + int SampleRateHz() const override; + size_t NumChannels() const override; + int RtpTimestampRateHz() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + + void Reset() override; + bool SetFec(bool enable) override; + + // Set Opus DTX. Once enabled, Opus stops transmission, when it detects + // voice being inactive. During that, it still sends 2 packets (one for + // content, one for signaling) about every 400 ms. + bool SetDtx(bool enable) override; + bool GetDtx() const override; + + bool SetApplication(Application application) override; + void SetMaxPlaybackRate(int frequency_hz) override; + bool EnableAudioNetworkAdaptor(const std::string& config_string, + RtcEventLog* event_log) override; + void DisableAudioNetworkAdaptor() override; + void OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) override; + void OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional bwe_period_ms) override; + void OnReceivedUplinkAllocation(BitrateAllocationUpdate update) override; + void OnReceivedRtt(int rtt_ms) override; + void OnReceivedOverhead(size_t overhead_bytes_per_packet) override; + void SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms) override; + ANAStats GetANAStats() const override; + absl::optional > GetFrameLengthRange() + const override; + rtc::ArrayView supported_frame_lengths_ms() const { + return config_.supported_frame_lengths_ms; + } + + // Getters for testing. + float packet_loss_rate() const { return packet_loss_rate_; } + AudioEncoderOpusConfig::ApplicationMode application() const { + return config_.application; + } + bool fec_enabled() const { return config_.fec_enabled; } + size_t num_channels_to_encode() const { return num_channels_to_encode_; } + int next_frame_length_ms() const { return next_frame_length_ms_; } + + protected: + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + private: + class PacketLossFractionSmoother; + + static absl::optional SdpToConfig( + const SdpAudioFormat& format); + static void AppendSupportedEncoders(std::vector* specs); + static AudioCodecInfo QueryAudioEncoder(const AudioEncoderOpusConfig& config); + static std::unique_ptr MakeAudioEncoder( + const AudioEncoderOpusConfig&, + int payload_type); + + size_t Num10msFramesPerPacket() const; + size_t SamplesPer10msFrame() const; + size_t SufficientOutputBufferSize() const; + bool RecreateEncoderInstance(const AudioEncoderOpusConfig& config); + void SetFrameLength(int frame_length_ms); + void SetNumChannelsToEncode(size_t num_channels_to_encode); + void SetProjectedPacketLossRate(float fraction); + + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional bwe_period_ms, + absl::optional link_capacity_allocation); + + // TODO(minyue): remove "override" when we can deprecate + // `AudioEncoder::SetTargetBitrate`. + void SetTargetBitrate(int target_bps) override; + + void ApplyAudioNetworkAdaptor(); + std::unique_ptr DefaultAudioNetworkAdaptorCreator( + absl::string_view config_string, + RtcEventLog* event_log) const; + + void MaybeUpdateUplinkBandwidth(); + + AudioEncoderOpusConfig config_; + const int payload_type_; + const bool use_stable_target_for_adaptation_; + const bool adjust_bandwidth_; + bool bitrate_changed_; + // A multiplier for bitrates at 5 kbps and higher. The target bitrate + // will be multiplied by these multipliers, each multiplier is applied to a + // 1 kbps range. + std::vector bitrate_multipliers_; + float packet_loss_rate_; + std::vector input_buffer_; + OpusEncInst* inst_; + uint32_t first_timestamp_in_buffer_; + size_t num_channels_to_encode_; + int next_frame_length_ms_; + int complexity_; + std::unique_ptr packet_loss_fraction_smoother_; + const AudioNetworkAdaptorCreator audio_network_adaptor_creator_; + std::unique_ptr audio_network_adaptor_; + absl::optional overhead_bytes_per_packet_; + const std::unique_ptr bitrate_smoother_; + absl::optional bitrate_smoother_last_update_time_; + int consecutive_dtx_frames_; + + friend struct AudioEncoderOpus; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc new file mode 100644 index 0000000000..a2ebe43bbe --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc @@ -0,0 +1,914 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/opus/audio_encoder_opus.h" + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "common_audio/mocks/mock_smoothing_filter.h" +#include "modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h" +#include "modules/audio_coding/codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "rtc_base/checks.h" +#include "rtc_base/fake_clock.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +using ::testing::NiceMock; +using ::testing::Return; + +namespace { + +constexpr int kDefaultOpusPayloadType = 105; +constexpr int kDefaultOpusRate = 32000; +constexpr int kDefaultOpusPacSize = 960; +constexpr int64_t kInitialTimeUs = 12345678; + +AudioEncoderOpusConfig CreateConfigWithParameters( + const SdpAudioFormat::Parameters& params) { + const SdpAudioFormat format("opus", 48000, 2, params); + return *AudioEncoderOpus::SdpToConfig(format); +} + +struct AudioEncoderOpusStates { + MockAudioNetworkAdaptor* mock_audio_network_adaptor; + MockSmoothingFilter* mock_bitrate_smoother; + std::unique_ptr encoder; + std::unique_ptr fake_clock; + AudioEncoderOpusConfig config; +}; + +std::unique_ptr CreateCodec(int sample_rate_hz, + size_t num_channels) { + std::unique_ptr states = + std::make_unique(); + states->mock_audio_network_adaptor = nullptr; + states->fake_clock.reset(new rtc::ScopedFakeClock()); + states->fake_clock->SetTime(Timestamp::Micros(kInitialTimeUs)); + + MockAudioNetworkAdaptor** mock_ptr = &states->mock_audio_network_adaptor; + AudioEncoderOpusImpl::AudioNetworkAdaptorCreator creator = + [mock_ptr](absl::string_view, RtcEventLog* event_log) { + std::unique_ptr adaptor( + new NiceMock()); + EXPECT_CALL(*adaptor, Die()); + *mock_ptr = adaptor.get(); + return adaptor; + }; + + AudioEncoderOpusConfig config; + config.frame_size_ms = rtc::CheckedDivExact(kDefaultOpusPacSize, 48); + config.sample_rate_hz = sample_rate_hz; + config.num_channels = num_channels; + config.bitrate_bps = kDefaultOpusRate; + config.application = num_channels == 1 + ? AudioEncoderOpusConfig::ApplicationMode::kVoip + : AudioEncoderOpusConfig::ApplicationMode::kAudio; + config.supported_frame_lengths_ms.push_back(config.frame_size_ms); + states->config = config; + + std::unique_ptr bitrate_smoother( + new MockSmoothingFilter()); + states->mock_bitrate_smoother = bitrate_smoother.get(); + + states->encoder.reset( + new AudioEncoderOpusImpl(states->config, kDefaultOpusPayloadType, creator, + std::move(bitrate_smoother))); + return states; +} + +AudioEncoderRuntimeConfig CreateEncoderRuntimeConfig() { + constexpr int kBitrate = 40000; + constexpr int kFrameLength = 60; + constexpr bool kEnableDtx = false; + constexpr size_t kNumChannels = 1; + AudioEncoderRuntimeConfig config; + config.bitrate_bps = kBitrate; + config.frame_length_ms = kFrameLength; + config.enable_dtx = kEnableDtx; + config.num_channels = kNumChannels; + return config; +} + +void CheckEncoderRuntimeConfig(const AudioEncoderOpusImpl* encoder, + const AudioEncoderRuntimeConfig& config) { + EXPECT_EQ(*config.bitrate_bps, encoder->GetTargetBitrate()); + EXPECT_EQ(*config.frame_length_ms, encoder->next_frame_length_ms()); + EXPECT_EQ(*config.enable_dtx, encoder->GetDtx()); + EXPECT_EQ(*config.num_channels, encoder->num_channels_to_encode()); +} + +// Create 10ms audio data blocks for a total packet size of "packet_size_ms". +std::unique_ptr Create10msAudioBlocks( + const std::unique_ptr& encoder, + int packet_size_ms) { + const std::string file_name = + test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + + std::unique_ptr speech_data(new test::AudioLoop()); + int audio_samples_per_ms = + rtc::CheckedDivExact(encoder->SampleRateHz(), 1000); + if (!speech_data->Init( + file_name, + packet_size_ms * audio_samples_per_ms * + encoder->num_channels_to_encode(), + 10 * audio_samples_per_ms * encoder->num_channels_to_encode())) + return nullptr; + return speech_data; +} + +} // namespace + +class AudioEncoderOpusTest : public ::testing::TestWithParam { + protected: + int sample_rate_hz_{GetParam()}; +}; +INSTANTIATE_TEST_SUITE_P(Param, + AudioEncoderOpusTest, + ::testing::Values(16000, 48000)); + +TEST_P(AudioEncoderOpusTest, DefaultApplicationModeMono) { + auto states = CreateCodec(sample_rate_hz_, 1); + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip, + states->encoder->application()); +} + +TEST_P(AudioEncoderOpusTest, DefaultApplicationModeStereo) { + auto states = CreateCodec(sample_rate_hz_, 2); + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kAudio, + states->encoder->application()); +} + +TEST_P(AudioEncoderOpusTest, ChangeApplicationMode) { + auto states = CreateCodec(sample_rate_hz_, 2); + EXPECT_TRUE( + states->encoder->SetApplication(AudioEncoder::Application::kSpeech)); + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip, + states->encoder->application()); +} + +TEST_P(AudioEncoderOpusTest, ResetWontChangeApplicationMode) { + auto states = CreateCodec(sample_rate_hz_, 2); + + // Trigger a reset. + states->encoder->Reset(); + // Verify that the mode is still kAudio. + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kAudio, + states->encoder->application()); + + // Now change to kVoip. + EXPECT_TRUE( + states->encoder->SetApplication(AudioEncoder::Application::kSpeech)); + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip, + states->encoder->application()); + + // Trigger a reset again. + states->encoder->Reset(); + // Verify that the mode is still kVoip. + EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip, + states->encoder->application()); +} + +TEST_P(AudioEncoderOpusTest, ToggleDtx) { + auto states = CreateCodec(sample_rate_hz_, 2); + // Enable DTX + EXPECT_TRUE(states->encoder->SetDtx(true)); + EXPECT_TRUE(states->encoder->GetDtx()); + // Turn off DTX. + EXPECT_TRUE(states->encoder->SetDtx(false)); + EXPECT_FALSE(states->encoder->GetDtx()); +} + +TEST_P(AudioEncoderOpusTest, + OnReceivedUplinkBandwidthWithoutAudioNetworkAdaptor) { + auto states = CreateCodec(sample_rate_hz_, 1); + // Constants are replicated from audio_states->encoderopus.cc. + const int kMinBitrateBps = 6000; + const int kMaxBitrateBps = 510000; + const int kOverheadBytesPerPacket = 64; + states->encoder->OnReceivedOverhead(kOverheadBytesPerPacket); + const int kOverheadBps = 8 * kOverheadBytesPerPacket * + rtc::CheckedDivExact(48000, kDefaultOpusPacSize); + // Set a too low bitrate. + states->encoder->OnReceivedUplinkBandwidth(kMinBitrateBps + kOverheadBps - 1, + absl::nullopt); + EXPECT_EQ(kMinBitrateBps, states->encoder->GetTargetBitrate()); + // Set a too high bitrate. + states->encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps + kOverheadBps + 1, + absl::nullopt); + EXPECT_EQ(kMaxBitrateBps, states->encoder->GetTargetBitrate()); + // Set the minimum rate. + states->encoder->OnReceivedUplinkBandwidth(kMinBitrateBps + kOverheadBps, + absl::nullopt); + EXPECT_EQ(kMinBitrateBps, states->encoder->GetTargetBitrate()); + // Set the maximum rate. + states->encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps + kOverheadBps, + absl::nullopt); + EXPECT_EQ(kMaxBitrateBps, states->encoder->GetTargetBitrate()); + // Set rates from kMaxBitrateBps up to 32000 bps. + for (int rate = kMinBitrateBps + kOverheadBps; rate <= 32000 + kOverheadBps; + rate += 1000) { + states->encoder->OnReceivedUplinkBandwidth(rate, absl::nullopt); + EXPECT_EQ(rate - kOverheadBps, states->encoder->GetTargetBitrate()); + } +} + +TEST_P(AudioEncoderOpusTest, SetReceiverFrameLengthRange) { + auto states = CreateCodec(sample_rate_hz_, 2); + // Before calling to `SetReceiverFrameLengthRange`, + // `supported_frame_lengths_ms` should contain only the frame length being + // used. + using ::testing::ElementsAre; + EXPECT_THAT(states->encoder->supported_frame_lengths_ms(), + ElementsAre(states->encoder->next_frame_length_ms())); + states->encoder->SetReceiverFrameLengthRange(0, 12345); + states->encoder->SetReceiverFrameLengthRange(21, 60); + EXPECT_THAT(states->encoder->supported_frame_lengths_ms(), + ElementsAre(40, 60)); + states->encoder->SetReceiverFrameLengthRange(20, 59); + EXPECT_THAT(states->encoder->supported_frame_lengths_ms(), + ElementsAre(20, 40)); +} + +TEST_P(AudioEncoderOpusTest, + InvokeAudioNetworkAdaptorOnReceivedUplinkPacketLossFraction) { + auto states = CreateCodec(sample_rate_hz_, 2); + states->encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any packet loss fraction is fine. + constexpr float kUplinkPacketLoss = 0.1f; + EXPECT_CALL(*states->mock_audio_network_adaptor, + SetUplinkPacketLossFraction(kUplinkPacketLoss)); + states->encoder->OnReceivedUplinkPacketLossFraction(kUplinkPacketLoss); + + CheckEncoderRuntimeConfig(states->encoder.get(), config); +} + +TEST_P(AudioEncoderOpusTest, + InvokeAudioNetworkAdaptorOnReceivedUplinkBandwidth) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-StableTargetAdaptation/Disabled/"); + auto states = CreateCodec(sample_rate_hz_, 2); + states->encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any target audio bitrate is fine. + constexpr int kTargetAudioBitrate = 30000; + constexpr int64_t kProbingIntervalMs = 3000; + EXPECT_CALL(*states->mock_audio_network_adaptor, + SetTargetAudioBitrate(kTargetAudioBitrate)); + EXPECT_CALL(*states->mock_bitrate_smoother, + SetTimeConstantMs(kProbingIntervalMs * 4)); + EXPECT_CALL(*states->mock_bitrate_smoother, AddSample(kTargetAudioBitrate)); + states->encoder->OnReceivedUplinkBandwidth(kTargetAudioBitrate, + kProbingIntervalMs); + + CheckEncoderRuntimeConfig(states->encoder.get(), config); +} + +TEST_P(AudioEncoderOpusTest, + InvokeAudioNetworkAdaptorOnReceivedUplinkAllocation) { + auto states = CreateCodec(sample_rate_hz_, 2); + states->encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + BitrateAllocationUpdate update; + update.target_bitrate = DataRate::BitsPerSec(30000); + update.stable_target_bitrate = DataRate::BitsPerSec(20000); + update.bwe_period = TimeDelta::Millis(200); + EXPECT_CALL(*states->mock_audio_network_adaptor, + SetTargetAudioBitrate(update.target_bitrate.bps())); + EXPECT_CALL(*states->mock_audio_network_adaptor, + SetUplinkBandwidth(update.stable_target_bitrate.bps())); + states->encoder->OnReceivedUplinkAllocation(update); + + CheckEncoderRuntimeConfig(states->encoder.get(), config); +} + +TEST_P(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedRtt) { + auto states = CreateCodec(sample_rate_hz_, 2); + states->encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any rtt is fine. + constexpr int kRtt = 30; + EXPECT_CALL(*states->mock_audio_network_adaptor, SetRtt(kRtt)); + states->encoder->OnReceivedRtt(kRtt); + + CheckEncoderRuntimeConfig(states->encoder.get(), config); +} + +TEST_P(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedOverhead) { + auto states = CreateCodec(sample_rate_hz_, 2); + states->encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any overhead is fine. + constexpr size_t kOverhead = 64; + EXPECT_CALL(*states->mock_audio_network_adaptor, SetOverhead(kOverhead)); + states->encoder->OnReceivedOverhead(kOverhead); + + CheckEncoderRuntimeConfig(states->encoder.get(), config); +} + +TEST_P(AudioEncoderOpusTest, + PacketLossFractionSmoothedOnSetUplinkPacketLossFraction) { + auto states = CreateCodec(sample_rate_hz_, 2); + + // The values are carefully chosen so that if no smoothing is made, the test + // will fail. + constexpr float kPacketLossFraction_1 = 0.02f; + constexpr float kPacketLossFraction_2 = 0.198f; + // `kSecondSampleTimeMs` is chosen to ease the calculation since + // 0.9999 ^ 6931 = 0.5. + constexpr int64_t kSecondSampleTimeMs = 6931; + + // First time, no filtering. + states->encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1); + EXPECT_FLOAT_EQ(0.02f, states->encoder->packet_loss_rate()); + + states->fake_clock->AdvanceTime(TimeDelta::Millis(kSecondSampleTimeMs)); + states->encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2); + + // Now the output of packet loss fraction smoother should be + // (0.02 + 0.198) / 2 = 0.109. + EXPECT_NEAR(0.109f, states->encoder->packet_loss_rate(), 0.001); +} + +TEST_P(AudioEncoderOpusTest, PacketLossRateUpperBounded) { + auto states = CreateCodec(sample_rate_hz_, 2); + + states->encoder->OnReceivedUplinkPacketLossFraction(0.5); + EXPECT_FLOAT_EQ(0.2f, states->encoder->packet_loss_rate()); +} + +TEST_P(AudioEncoderOpusTest, DoNotInvokeSetTargetBitrateIfOverheadUnknown) { + auto states = CreateCodec(sample_rate_hz_, 2); + + states->encoder->OnReceivedUplinkBandwidth(kDefaultOpusRate * 2, + absl::nullopt); + + // Since `OnReceivedOverhead` has not been called, the codec bitrate should + // not change. + EXPECT_EQ(kDefaultOpusRate, states->encoder->GetTargetBitrate()); +} + +// Verifies that the complexity adaptation in the config works as intended. +TEST(AudioEncoderOpusTest, ConfigComplexityAdaptation) { + AudioEncoderOpusConfig config; + config.low_rate_complexity = 8; + config.complexity = 6; + + // Bitrate within hysteresis window. Expect empty output. + config.bitrate_bps = 12500; + EXPECT_EQ(absl::nullopt, AudioEncoderOpusImpl::GetNewComplexity(config)); + + // Bitrate below hysteresis window. Expect higher complexity. + config.bitrate_bps = 10999; + EXPECT_EQ(8, AudioEncoderOpusImpl::GetNewComplexity(config)); + + // Bitrate within hysteresis window. Expect empty output. + config.bitrate_bps = 12500; + EXPECT_EQ(absl::nullopt, AudioEncoderOpusImpl::GetNewComplexity(config)); + + // Bitrate above hysteresis window. Expect lower complexity. + config.bitrate_bps = 14001; + EXPECT_EQ(6, AudioEncoderOpusImpl::GetNewComplexity(config)); +} + +// Verifies that the bandwidth adaptation in the config works as intended. +TEST_P(AudioEncoderOpusTest, ConfigBandwidthAdaptation) { + AudioEncoderOpusConfig config; + const size_t opus_rate_khz = rtc::CheckedDivExact(sample_rate_hz_, 1000); + const std::vector silence( + opus_rate_khz * config.frame_size_ms * config.num_channels, 0); + constexpr size_t kMaxBytes = 1000; + uint8_t bitstream[kMaxBytes]; + + OpusEncInst* inst; + EXPECT_EQ(0, WebRtcOpus_EncoderCreate( + &inst, config.num_channels, + config.application == + AudioEncoderOpusConfig::ApplicationMode::kVoip + ? 0 + : 1, + sample_rate_hz_)); + + // Bitrate below minmum wideband. Expect narrowband. + config.bitrate_bps = absl::optional(7999); + auto bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(absl::optional(OPUS_BANDWIDTH_NARROWBAND), bandwidth); + WebRtcOpus_SetBandwidth(inst, *bandwidth); + // It is necessary to encode here because Opus has some logic in the encoder + // that goes from the user-set bandwidth to the used and returned one. + WebRtcOpus_Encode(inst, silence.data(), + rtc::CheckedDivExact(silence.size(), config.num_channels), + kMaxBytes, bitstream); + + // Bitrate not yet above maximum narrowband. Expect empty. + config.bitrate_bps = absl::optional(9000); + bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(absl::optional(), bandwidth); + + // Bitrate above maximum narrowband. Expect wideband. + config.bitrate_bps = absl::optional(9001); + bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(absl::optional(OPUS_BANDWIDTH_WIDEBAND), bandwidth); + WebRtcOpus_SetBandwidth(inst, *bandwidth); + // It is necessary to encode here because Opus has some logic in the encoder + // that goes from the user-set bandwidth to the used and returned one. + WebRtcOpus_Encode(inst, silence.data(), + rtc::CheckedDivExact(silence.size(), config.num_channels), + kMaxBytes, bitstream); + + // Bitrate not yet below minimum wideband. Expect empty. + config.bitrate_bps = absl::optional(8000); + bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(absl::optional(), bandwidth); + + // Bitrate above automatic threshold. Expect automatic. + config.bitrate_bps = absl::optional(12001); + bandwidth = AudioEncoderOpusImpl::GetNewBandwidth(config, inst); + EXPECT_EQ(absl::optional(OPUS_AUTO), bandwidth); + + EXPECT_EQ(0, WebRtcOpus_EncoderFree(inst)); +} + +TEST_P(AudioEncoderOpusTest, EmptyConfigDoesNotAffectEncoderSettings) { + auto states = CreateCodec(sample_rate_hz_, 2); + states->encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + AudioEncoderRuntimeConfig empty_config; + + EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)) + .WillOnce(Return(empty_config)); + + constexpr size_t kOverhead = 64; + EXPECT_CALL(*states->mock_audio_network_adaptor, SetOverhead(kOverhead)) + .Times(2); + states->encoder->OnReceivedOverhead(kOverhead); + states->encoder->OnReceivedOverhead(kOverhead); + + CheckEncoderRuntimeConfig(states->encoder.get(), config); +} + +TEST_P(AudioEncoderOpusTest, UpdateUplinkBandwidthInAudioNetworkAdaptor) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-StableTargetAdaptation/Disabled/"); + auto states = CreateCodec(sample_rate_hz_, 2); + states->encoder->EnableAudioNetworkAdaptor("", nullptr); + const size_t opus_rate_khz = rtc::CheckedDivExact(sample_rate_hz_, 1000); + const std::vector audio(opus_rate_khz * 10 * 2, 0); + rtc::Buffer encoded; + EXPECT_CALL(*states->mock_bitrate_smoother, GetAverage()) + .WillOnce(Return(50000)); + EXPECT_CALL(*states->mock_audio_network_adaptor, SetUplinkBandwidth(50000)); + states->encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + + // Repeat update uplink bandwidth tests. + for (int i = 0; i < 5; i++) { + // Don't update till it is time to update again. + states->fake_clock->AdvanceTime(TimeDelta::Millis( + states->config.uplink_bandwidth_update_interval_ms - 1)); + states->encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + + // Update when it is time to update. + EXPECT_CALL(*states->mock_bitrate_smoother, GetAverage()) + .WillOnce(Return(40000)); + EXPECT_CALL(*states->mock_audio_network_adaptor, SetUplinkBandwidth(40000)); + states->fake_clock->AdvanceTime(TimeDelta::Millis(1)); + states->encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + } +} + +TEST_P(AudioEncoderOpusTest, EncodeAtMinBitrate) { + auto states = CreateCodec(sample_rate_hz_, 1); + constexpr int kNumPacketsToEncode = 2; + auto audio_frames = + Create10msAudioBlocks(states->encoder, kNumPacketsToEncode * 20); + ASSERT_TRUE(audio_frames) << "Create10msAudioBlocks failed"; + rtc::Buffer encoded; + uint32_t rtp_timestamp = 12345; // Just a number not important to this test. + + states->encoder->OnReceivedUplinkBandwidth(0, absl::nullopt); + for (int packet_index = 0; packet_index < kNumPacketsToEncode; + packet_index++) { + // Make sure we are not encoding before we have enough data for + // a 20ms packet. + for (int index = 0; index < 1; index++) { + states->encoder->Encode(rtp_timestamp, audio_frames->GetNextBlock(), + &encoded); + EXPECT_EQ(0u, encoded.size()); + } + + // Should encode now. + states->encoder->Encode(rtp_timestamp, audio_frames->GetNextBlock(), + &encoded); + EXPECT_GT(encoded.size(), 0u); + encoded.Clear(); + } +} + +TEST(AudioEncoderOpusTest, TestConfigDefaults) { + const auto config_opt = AudioEncoderOpus::SdpToConfig({"opus", 48000, 2}); + ASSERT_TRUE(config_opt); + EXPECT_EQ(48000, config_opt->max_playback_rate_hz); + EXPECT_EQ(1u, config_opt->num_channels); + EXPECT_FALSE(config_opt->fec_enabled); + EXPECT_FALSE(config_opt->dtx_enabled); + EXPECT_EQ(20, config_opt->frame_size_ms); +} + +TEST(AudioEncoderOpusTest, TestConfigFromParams) { + const auto config1 = CreateConfigWithParameters({{"stereo", "0"}}); + EXPECT_EQ(1U, config1.num_channels); + + const auto config2 = CreateConfigWithParameters({{"stereo", "1"}}); + EXPECT_EQ(2U, config2.num_channels); + + const auto config3 = CreateConfigWithParameters({{"useinbandfec", "0"}}); + EXPECT_FALSE(config3.fec_enabled); + + const auto config4 = CreateConfigWithParameters({{"useinbandfec", "1"}}); + EXPECT_TRUE(config4.fec_enabled); + + const auto config5 = CreateConfigWithParameters({{"usedtx", "0"}}); + EXPECT_FALSE(config5.dtx_enabled); + + const auto config6 = CreateConfigWithParameters({{"usedtx", "1"}}); + EXPECT_TRUE(config6.dtx_enabled); + + const auto config7 = CreateConfigWithParameters({{"cbr", "0"}}); + EXPECT_FALSE(config7.cbr_enabled); + + const auto config8 = CreateConfigWithParameters({{"cbr", "1"}}); + EXPECT_TRUE(config8.cbr_enabled); + + const auto config9 = + CreateConfigWithParameters({{"maxplaybackrate", "12345"}}); + EXPECT_EQ(12345, config9.max_playback_rate_hz); + + const auto config10 = + CreateConfigWithParameters({{"maxaveragebitrate", "96000"}}); + EXPECT_EQ(96000, config10.bitrate_bps); + + const auto config11 = CreateConfigWithParameters({{"maxptime", "40"}}); + for (int frame_length : config11.supported_frame_lengths_ms) { + EXPECT_LE(frame_length, 40); + } + + const auto config12 = CreateConfigWithParameters({{"minptime", "40"}}); + for (int frame_length : config12.supported_frame_lengths_ms) { + EXPECT_GE(frame_length, 40); + } + + const auto config13 = CreateConfigWithParameters({{"ptime", "40"}}); + EXPECT_EQ(40, config13.frame_size_ms); + + constexpr int kMinSupportedFrameLength = 10; + constexpr int kMaxSupportedFrameLength = + WEBRTC_OPUS_SUPPORT_120MS_PTIME ? 120 : 60; + + const auto config14 = CreateConfigWithParameters({{"ptime", "1"}}); + EXPECT_EQ(kMinSupportedFrameLength, config14.frame_size_ms); + + const auto config15 = CreateConfigWithParameters({{"ptime", "2000"}}); + EXPECT_EQ(kMaxSupportedFrameLength, config15.frame_size_ms); +} + +TEST(AudioEncoderOpusTest, TestConfigFromInvalidParams) { + const webrtc::SdpAudioFormat format("opus", 48000, 2); + const auto default_config = *AudioEncoderOpus::SdpToConfig(format); +#if WEBRTC_OPUS_SUPPORT_120MS_PTIME + const std::vector default_supported_frame_lengths_ms({20, 40, 60, 120}); +#else + const std::vector default_supported_frame_lengths_ms({20, 40, 60}); +#endif + + AudioEncoderOpusConfig config; + config = CreateConfigWithParameters({{"stereo", "invalid"}}); + EXPECT_EQ(default_config.num_channels, config.num_channels); + + config = CreateConfigWithParameters({{"useinbandfec", "invalid"}}); + EXPECT_EQ(default_config.fec_enabled, config.fec_enabled); + + config = CreateConfigWithParameters({{"usedtx", "invalid"}}); + EXPECT_EQ(default_config.dtx_enabled, config.dtx_enabled); + + config = CreateConfigWithParameters({{"cbr", "invalid"}}); + EXPECT_EQ(default_config.dtx_enabled, config.dtx_enabled); + + config = CreateConfigWithParameters({{"maxplaybackrate", "0"}}); + EXPECT_EQ(default_config.max_playback_rate_hz, config.max_playback_rate_hz); + + config = CreateConfigWithParameters({{"maxplaybackrate", "-23"}}); + EXPECT_EQ(default_config.max_playback_rate_hz, config.max_playback_rate_hz); + + config = CreateConfigWithParameters({{"maxplaybackrate", "not a number!"}}); + EXPECT_EQ(default_config.max_playback_rate_hz, config.max_playback_rate_hz); + + config = CreateConfigWithParameters({{"maxaveragebitrate", "0"}}); + EXPECT_EQ(6000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxaveragebitrate", "-1000"}}); + EXPECT_EQ(6000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxaveragebitrate", "1024000"}}); + EXPECT_EQ(510000, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxaveragebitrate", "not a number!"}}); + EXPECT_EQ(default_config.bitrate_bps, config.bitrate_bps); + + config = CreateConfigWithParameters({{"maxptime", "invalid"}}); + EXPECT_EQ(default_supported_frame_lengths_ms, + config.supported_frame_lengths_ms); + + config = CreateConfigWithParameters({{"minptime", "invalid"}}); + EXPECT_EQ(default_supported_frame_lengths_ms, + config.supported_frame_lengths_ms); + + config = CreateConfigWithParameters({{"ptime", "invalid"}}); + EXPECT_EQ(default_supported_frame_lengths_ms, + config.supported_frame_lengths_ms); +} + +TEST(AudioEncoderOpusTest, GetFrameLenghtRange) { + AudioEncoderOpusConfig config = + CreateConfigWithParameters({{"maxptime", "10"}, {"ptime", "10"}}); + std::unique_ptr encoder = + AudioEncoderOpus::MakeAudioEncoder(config, kDefaultOpusPayloadType); + auto ptime = webrtc::TimeDelta::Millis(10); + absl::optional> range = { + {ptime, ptime}}; + EXPECT_EQ(encoder->GetFrameLengthRange(), range); +} + +// Test that bitrate will be overridden by the "maxaveragebitrate" parameter. +// Also test that the "maxaveragebitrate" can't be set to values outside the +// range of 6000 and 510000 +TEST(AudioEncoderOpusTest, SetSendCodecOpusMaxAverageBitrate) { + // Ignore if less than 6000. + const auto config1 = AudioEncoderOpus::SdpToConfig( + {"opus", 48000, 2, {{"maxaveragebitrate", "5999"}}}); + EXPECT_EQ(6000, config1->bitrate_bps); + + // Ignore if larger than 510000. + const auto config2 = AudioEncoderOpus::SdpToConfig( + {"opus", 48000, 2, {{"maxaveragebitrate", "510001"}}}); + EXPECT_EQ(510000, config2->bitrate_bps); + + const auto config3 = AudioEncoderOpus::SdpToConfig( + {"opus", 48000, 2, {{"maxaveragebitrate", "200000"}}}); + EXPECT_EQ(200000, config3->bitrate_bps); +} + +// Test maxplaybackrate <= 8000 triggers Opus narrow band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateNb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "8000"}}); + EXPECT_EQ(8000, config.max_playback_rate_hz); + EXPECT_EQ(12000, config.bitrate_bps); + + config = CreateConfigWithParameters( + {{"maxplaybackrate", "8000"}, {"stereo", "1"}}); + EXPECT_EQ(8000, config.max_playback_rate_hz); + EXPECT_EQ(24000, config.bitrate_bps); +} + +// Test 8000 < maxplaybackrate <= 12000 triggers Opus medium band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateMb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "8001"}}); + EXPECT_EQ(8001, config.max_playback_rate_hz); + EXPECT_EQ(20000, config.bitrate_bps); + + config = CreateConfigWithParameters( + {{"maxplaybackrate", "8001"}, {"stereo", "1"}}); + EXPECT_EQ(8001, config.max_playback_rate_hz); + EXPECT_EQ(40000, config.bitrate_bps); +} + +// Test 12000 < maxplaybackrate <= 16000 triggers Opus wide band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateWb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "12001"}}); + EXPECT_EQ(12001, config.max_playback_rate_hz); + EXPECT_EQ(20000, config.bitrate_bps); + + config = CreateConfigWithParameters( + {{"maxplaybackrate", "12001"}, {"stereo", "1"}}); + EXPECT_EQ(12001, config.max_playback_rate_hz); + EXPECT_EQ(40000, config.bitrate_bps); +} + +// Test 16000 < maxplaybackrate <= 24000 triggers Opus super wide band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateSwb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "16001"}}); + EXPECT_EQ(16001, config.max_playback_rate_hz); + EXPECT_EQ(32000, config.bitrate_bps); + + config = CreateConfigWithParameters( + {{"maxplaybackrate", "16001"}, {"stereo", "1"}}); + EXPECT_EQ(16001, config.max_playback_rate_hz); + EXPECT_EQ(64000, config.bitrate_bps); +} + +// Test 24000 < maxplaybackrate triggers Opus full band mode. +TEST(AudioEncoderOpusTest, SetMaxPlaybackRateFb) { + auto config = CreateConfigWithParameters({{"maxplaybackrate", "24001"}}); + EXPECT_EQ(24001, config.max_playback_rate_hz); + EXPECT_EQ(32000, config.bitrate_bps); + + config = CreateConfigWithParameters( + {{"maxplaybackrate", "24001"}, {"stereo", "1"}}); + EXPECT_EQ(24001, config.max_playback_rate_hz); + EXPECT_EQ(64000, config.bitrate_bps); +} + +TEST_P(AudioEncoderOpusTest, OpusFlagDtxAsNonSpeech) { + // Create encoder with DTX enabled. + AudioEncoderOpusConfig config; + config.dtx_enabled = true; + config.sample_rate_hz = sample_rate_hz_; + constexpr int payload_type = 17; + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(config, payload_type); + + // Open file containing speech and silence. + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + test::AudioLoop audio_loop; + // Use the file as if it were sampled at our desired input rate. + const size_t max_loop_length_samples = + sample_rate_hz_ * 10; // Max 10 second loop. + const size_t input_block_size_samples = + 10 * sample_rate_hz_ / 1000; // 10 ms. + EXPECT_TRUE(audio_loop.Init(kInputFileName, max_loop_length_samples, + input_block_size_samples)); + + // Encode. + AudioEncoder::EncodedInfo info; + rtc::Buffer encoded(500); + int nonspeech_frames = 0; + int max_nonspeech_frames = 0; + int dtx_frames = 0; + int max_dtx_frames = 0; + uint32_t rtp_timestamp = 0u; + for (size_t i = 0; i < 500; ++i) { + encoded.Clear(); + + // Every second call to the encoder will generate an Opus packet. + for (int j = 0; j < 2; j++) { + info = + encoder->Encode(rtp_timestamp, audio_loop.GetNextBlock(), &encoded); + rtp_timestamp += input_block_size_samples; + } + + // Bookkeeping of number of DTX frames. + if (info.encoded_bytes <= 2) { + ++dtx_frames; + } else { + if (dtx_frames > max_dtx_frames) + max_dtx_frames = dtx_frames; + dtx_frames = 0; + } + + // Bookkeeping of number of non-speech frames. + if (info.speech == 0) { + ++nonspeech_frames; + } else { + if (nonspeech_frames > max_nonspeech_frames) + max_nonspeech_frames = nonspeech_frames; + nonspeech_frames = 0; + } + } + + // Maximum number of consecutive non-speech packets should exceed 15. + EXPECT_GT(max_nonspeech_frames, 15); +} + +TEST(AudioEncoderOpusTest, OpusDtxFilteringHighEnergyRefreshPackets) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-OpusAvoidNoisePumpingDuringDtx/Enabled/"); + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/testfile16kHz", "pcm"); + constexpr int kSampleRateHz = 16000; + AudioEncoderOpusConfig config; + config.dtx_enabled = true; + config.sample_rate_hz = kSampleRateHz; + constexpr int payload_type = 17; + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(config, payload_type); + test::AudioLoop audio_loop; + constexpr size_t kMaxLoopLengthSaples = kSampleRateHz * 11.6f; + constexpr size_t kInputBlockSizeSamples = kSampleRateHz / 100; + EXPECT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSaples, + kInputBlockSizeSamples)); + AudioEncoder::EncodedInfo info; + rtc::Buffer encoded(500); + // Encode the audio file and store the last part that corresponds to silence. + constexpr size_t kSilenceDurationSamples = kSampleRateHz * 0.2f; + std::array silence; + uint32_t rtp_timestamp = 0; + bool last_packet_dtx_frame = false; + bool opus_entered_dtx = false; + bool silence_filled = false; + size_t timestamp_start_silence = 0; + while (!silence_filled && rtp_timestamp < kMaxLoopLengthSaples) { + encoded.Clear(); + // Every second call to the encoder will generate an Opus packet. + for (int j = 0; j < 2; j++) { + auto next_frame = audio_loop.GetNextBlock(); + info = encoder->Encode(rtp_timestamp, next_frame, &encoded); + if (opus_entered_dtx) { + size_t silence_frame_start = rtp_timestamp - timestamp_start_silence; + silence_filled = silence_frame_start >= kSilenceDurationSamples; + if (!silence_filled) { + std::copy(next_frame.begin(), next_frame.end(), + silence.begin() + silence_frame_start); + } + } + rtp_timestamp += kInputBlockSizeSamples; + } + EXPECT_TRUE(info.encoded_bytes > 0 || last_packet_dtx_frame); + last_packet_dtx_frame = info.encoded_bytes > 0 ? info.encoded_bytes <= 2 + : last_packet_dtx_frame; + if (info.encoded_bytes <= 2 && !opus_entered_dtx) { + timestamp_start_silence = rtp_timestamp; + } + opus_entered_dtx = info.encoded_bytes <= 2; + } + + EXPECT_TRUE(silence_filled); + // The copied 200 ms of silence is used for creating 6 bursts that are fed to + // the encoder, the first three ones with a larger energy and the last three + // with a lower energy. This test verifies that the encoder just sends refresh + // DTX packets during the last bursts. + int number_non_empty_packets_during_increase = 0; + int number_non_empty_packets_during_decrease = 0; + for (size_t burst = 0; burst < 6; ++burst) { + uint32_t rtp_timestamp_start = rtp_timestamp; + const bool increase_noise = burst < 3; + const float gain = increase_noise ? 1.4f : 0.0f; + while (rtp_timestamp < rtp_timestamp_start + kSilenceDurationSamples) { + encoded.Clear(); + // Every second call to the encoder will generate an Opus packet. + for (int j = 0; j < 2; j++) { + std::array silence_frame; + size_t silence_frame_start = rtp_timestamp - rtp_timestamp_start; + std::transform( + silence.begin() + silence_frame_start, + silence.begin() + silence_frame_start + kInputBlockSizeSamples, + silence_frame.begin(), [gain](float s) { return gain * s; }); + info = encoder->Encode(rtp_timestamp, silence_frame, &encoded); + rtp_timestamp += kInputBlockSizeSamples; + } + EXPECT_TRUE(info.encoded_bytes > 0 || last_packet_dtx_frame); + last_packet_dtx_frame = info.encoded_bytes > 0 ? info.encoded_bytes <= 2 + : last_packet_dtx_frame; + // Tracking the number of non empty packets. + if (increase_noise && info.encoded_bytes > 2) { + number_non_empty_packets_during_increase++; + } + if (!increase_noise && info.encoded_bytes > 2) { + number_non_empty_packets_during_decrease++; + } + } + } + // Check that the refresh DTX packets are just sent during the decrease energy + // region. + EXPECT_EQ(number_non_empty_packets_during_increase, 0); + EXPECT_GT(number_non_empty_packets_during_decrease, 0); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_bandwidth_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_bandwidth_unittest.cc new file mode 100644 index 0000000000..38b60c6187 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_bandwidth_unittest.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/opus/audio_decoder_opus.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "common_audio/include/audio_util.h" +#include "common_audio/window_generator.h" +#include "modules/audio_coding/codecs/opus/test/lapped_transform.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace { + +constexpr size_t kNumChannels = 1u; +constexpr int kSampleRateHz = 48000; +constexpr size_t kMaxLoopLengthSamples = kSampleRateHz * 50; // 50 seconds. +constexpr size_t kInputBlockSizeSamples = 10 * kSampleRateHz / 1000; // 10 ms +constexpr size_t kOutputBlockSizeSamples = 20 * kSampleRateHz / 1000; // 20 ms +constexpr size_t kFftSize = 1024; +constexpr size_t kNarrowbandSize = 4000 * kFftSize / kSampleRateHz; +constexpr float kKbdAlpha = 1.5f; + +class PowerRatioEstimator : public LappedTransform::Callback { + public: + PowerRatioEstimator() : low_pow_(0.f), high_pow_(0.f) { + WindowGenerator::KaiserBesselDerived(kKbdAlpha, kFftSize, window_); + transform_.reset(new LappedTransform(kNumChannels, 0u, + kInputBlockSizeSamples, window_, + kFftSize, kFftSize / 2, this)); + } + + void ProcessBlock(float* data) { transform_->ProcessChunk(&data, nullptr); } + + float PowerRatio() { return high_pow_ / low_pow_; } + + protected: + void ProcessAudioBlock(const std::complex* const* input, + size_t num_input_channels, + size_t num_freq_bins, + size_t num_output_channels, + std::complex* const* output) override { + float low_pow = 0.f; + float high_pow = 0.f; + for (size_t i = 0u; i < num_input_channels; ++i) { + for (size_t j = 0u; j < kNarrowbandSize; ++j) { + float low_mag = std::abs(input[i][j]); + low_pow += low_mag * low_mag; + float high_mag = std::abs(input[i][j + kNarrowbandSize]); + high_pow += high_mag * high_mag; + } + } + low_pow_ += low_pow / (num_input_channels * kFftSize); + high_pow_ += high_pow / (num_input_channels * kFftSize); + } + + private: + std::unique_ptr transform_; + float window_[kFftSize]; + float low_pow_; + float high_pow_; +}; + +float EncodedPowerRatio(AudioEncoder* encoder, + AudioDecoder* decoder, + test::AudioLoop* audio_loop) { + // Encode and decode. + uint32_t rtp_timestamp = 0u; + constexpr size_t kBufferSize = 500; + rtc::Buffer encoded(kBufferSize); + std::vector decoded(kOutputBlockSizeSamples); + std::vector decoded_float(kOutputBlockSizeSamples); + AudioDecoder::SpeechType speech_type = AudioDecoder::kSpeech; + PowerRatioEstimator power_ratio_estimator; + for (size_t i = 0; i < 1000; ++i) { + encoded.Clear(); + AudioEncoder::EncodedInfo encoder_info = + encoder->Encode(rtp_timestamp, audio_loop->GetNextBlock(), &encoded); + rtp_timestamp += kInputBlockSizeSamples; + if (encoded.size() > 0) { + int decoder_info = decoder->Decode( + encoded.data(), encoded.size(), kSampleRateHz, + decoded.size() * sizeof(decoded[0]), decoded.data(), &speech_type); + if (decoder_info > 0) { + S16ToFloat(decoded.data(), decoded.size(), decoded_float.data()); + power_ratio_estimator.ProcessBlock(decoded_float.data()); + } + } + } + return power_ratio_estimator.PowerRatio(); +} + +} // namespace + +// TODO(ivoc): Remove this test, WebRTC-AdjustOpusBandwidth is obsolete. +TEST(BandwidthAdaptationTest, BandwidthAdaptationTest) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-AdjustOpusBandwidth/Enabled/"); + + constexpr float kMaxNarrowbandRatio = 0.0035f; + constexpr float kMinWidebandRatio = 0.01f; + + // Create encoder. + AudioEncoderOpusConfig enc_config; + enc_config.bitrate_bps = absl::optional(7999); + enc_config.num_channels = kNumChannels; + constexpr int payload_type = 17; + auto encoder = AudioEncoderOpus::MakeAudioEncoder(enc_config, payload_type); + + // Create decoder. + AudioDecoderOpus::Config dec_config; + dec_config.num_channels = kNumChannels; + auto decoder = AudioDecoderOpus::MakeAudioDecoder(dec_config); + + // Open speech file. + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/speech_mono_32_48kHz", "pcm"); + test::AudioLoop audio_loop; + EXPECT_EQ(kSampleRateHz, encoder->SampleRateHz()); + ASSERT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)); + + EXPECT_LT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMaxNarrowbandRatio); + + encoder->OnReceivedTargetAudioBitrate(9000); + EXPECT_LT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMaxNarrowbandRatio); + + encoder->OnReceivedTargetAudioBitrate(9001); + EXPECT_GT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMinWidebandRatio); + + encoder->OnReceivedTargetAudioBitrate(8000); + EXPECT_GT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMinWidebandRatio); + + encoder->OnReceivedTargetAudioBitrate(12001); + EXPECT_GT(EncodedPowerRatio(encoder.get(), decoder.get(), &audio_loop), + kMinWidebandRatio); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc new file mode 100644 index 0000000000..e8c131092c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/metrics/metric.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "rtc_base/time_utils.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace { + +using ::webrtc::test::GetGlobalMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + +int64_t RunComplexityTest(const AudioEncoderOpusConfig& config) { + // Create encoder. + constexpr int payload_type = 17; + const auto encoder = AudioEncoderOpus::MakeAudioEncoder(config, payload_type); + // Open speech file. + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/speech_mono_32_48kHz", "pcm"); + test::AudioLoop audio_loop; + constexpr int kSampleRateHz = 48000; + EXPECT_EQ(kSampleRateHz, encoder->SampleRateHz()); + constexpr size_t kMaxLoopLengthSamples = + kSampleRateHz * 10; // 10 second loop. + constexpr size_t kInputBlockSizeSamples = + 10 * kSampleRateHz / 1000; // 60 ms. + EXPECT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)); + // Encode. + const int64_t start_time_ms = rtc::TimeMillis(); + AudioEncoder::EncodedInfo info; + rtc::Buffer encoded(500); + uint32_t rtp_timestamp = 0u; + for (size_t i = 0; i < 10000; ++i) { + encoded.Clear(); + info = encoder->Encode(rtp_timestamp, audio_loop.GetNextBlock(), &encoded); + rtp_timestamp += kInputBlockSizeSamples; + } + return rtc::TimeMillis() - start_time_ms; +} + +// This test encodes an audio file using Opus twice with different bitrates +// (~11 kbps and 15.5 kbps). The runtime for each is measured, and the ratio +// between the two is calculated and tracked. This test explicitly sets the +// low_rate_complexity to 9. When running on desktop platforms, this is the same +// as the regular complexity, and the expectation is that the resulting ratio +// should be less than 100% (since the encoder runs faster at lower bitrates, +// given a fixed complexity setting). On the other hand, when running on +// mobiles, the regular complexity is 5, and we expect the resulting ratio to +// be higher, since we have explicitly asked for a higher complexity setting at +// the lower rate. +TEST(AudioEncoderOpusComplexityAdaptationTest, Adaptation_On) { + // Create config. + AudioEncoderOpusConfig config; + // The limit -- including the hysteresis window -- at which the complexity + // shuold be increased. + config.bitrate_bps = 11000 - 1; + config.low_rate_complexity = 9; + int64_t runtime_10999bps = RunComplexityTest(config); + + config.bitrate_bps = 15500; + int64_t runtime_15500bps = RunComplexityTest(config); + + GetGlobalMetricsLogger()->LogSingleValueMetric( + "opus_encoding_complexity_ratio", "adaptation_on", + 100.0 * runtime_10999bps / runtime_15500bps, Unit::kPercent, + ImprovementDirection::kNeitherIsBetter); +} + +// This test is identical to the one above, but without the complexity +// adaptation enabled (neither on desktop, nor on mobile). The expectation is +// that the resulting ratio is less than 100% at all times. +TEST(AudioEncoderOpusComplexityAdaptationTest, Adaptation_Off) { + // Create config. + AudioEncoderOpusConfig config; + // The limit -- including the hysteresis window -- at which the complexity + // shuold be increased (but not in this test since complexity adaptation is + // disabled). + config.bitrate_bps = 11000 - 1; + int64_t runtime_10999bps = RunComplexityTest(config); + + config.bitrate_bps = 15500; + int64_t runtime_15500bps = RunComplexityTest(config); + + GetGlobalMetricsLogger()->LogSingleValueMetric( + "opus_encoding_complexity_ratio", "adaptation_off", + 100.0 * runtime_10999bps / runtime_15500bps, Unit::kPercent, + ImprovementDirection::kNeitherIsBetter); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc new file mode 100644 index 0000000000..815f26e31c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_fec_test.cc @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +using std::get; +using std::string; +using std::tuple; +using ::testing::TestWithParam; + +namespace webrtc { + +// Define coding parameter as . +typedef tuple coding_param; +typedef struct mode mode; + +struct mode { + bool fec; + uint8_t target_packet_loss_rate; +}; + +const int kOpusBlockDurationMs = 20; +const int kOpusSamplingKhz = 48; + +class OpusFecTest : public TestWithParam { + protected: + OpusFecTest(); + + void SetUp() override; + void TearDown() override; + + virtual void EncodeABlock(); + + virtual void DecodeABlock(bool lost_previous, bool lost_current); + + int block_duration_ms_; + int sampling_khz_; + size_t block_length_sample_; + + size_t channels_; + int bit_rate_; + + size_t data_pointer_; + size_t loop_length_samples_; + size_t max_bytes_; + size_t encoded_bytes_; + + WebRtcOpusEncInst* opus_encoder_; + WebRtcOpusDecInst* opus_decoder_; + + string in_filename_; + + std::unique_ptr in_data_; + std::unique_ptr out_data_; + std::unique_ptr bit_stream_; +}; + +void OpusFecTest::SetUp() { + channels_ = get<0>(GetParam()); + bit_rate_ = get<1>(GetParam()); + printf("Coding %zu channel signal at %d bps.\n", channels_, bit_rate_); + + in_filename_ = test::ResourcePath(get<2>(GetParam()), get<3>(GetParam())); + + FILE* fp = fopen(in_filename_.c_str(), "rb"); + ASSERT_FALSE(fp == NULL); + + // Obtain file size. + fseek(fp, 0, SEEK_END); + loop_length_samples_ = ftell(fp) / sizeof(int16_t); + rewind(fp); + + // Allocate memory to contain the whole file. + in_data_.reset( + new int16_t[loop_length_samples_ + block_length_sample_ * channels_]); + + // Copy the file into the buffer. + ASSERT_EQ(fread(&in_data_[0], sizeof(int16_t), loop_length_samples_, fp), + loop_length_samples_); + fclose(fp); + + // The audio will be used in a looped manner. To ease the acquisition of an + // audio frame that crosses the end of the excerpt, we add an extra block + // length of samples to the end of the array, starting over again from the + // beginning of the array. Audio frames cross the end of the excerpt always + // appear as a continuum of memory. + memcpy(&in_data_[loop_length_samples_], &in_data_[0], + block_length_sample_ * channels_ * sizeof(int16_t)); + + // Maximum number of bytes in output bitstream. + max_bytes_ = block_length_sample_ * channels_ * sizeof(int16_t); + + out_data_.reset(new int16_t[2 * block_length_sample_ * channels_]); + bit_stream_.reset(new uint8_t[max_bytes_]); + + // If channels_ == 1, use Opus VOIP mode, otherwise, audio mode. + int app = channels_ == 1 ? 0 : 1; + + // Create encoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, app, 48000)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_, 48000)); + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, bit_rate_)); +} + +void OpusFecTest::TearDown() { + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +OpusFecTest::OpusFecTest() + : block_duration_ms_(kOpusBlockDurationMs), + sampling_khz_(kOpusSamplingKhz), + block_length_sample_( + static_cast(block_duration_ms_ * sampling_khz_)), + data_pointer_(0), + max_bytes_(0), + encoded_bytes_(0), + opus_encoder_(NULL), + opus_decoder_(NULL) {} + +void OpusFecTest::EncodeABlock() { + int value = + WebRtcOpus_Encode(opus_encoder_, &in_data_[data_pointer_], + block_length_sample_, max_bytes_, &bit_stream_[0]); + EXPECT_GT(value, 0); + + encoded_bytes_ = static_cast(value); +} + +void OpusFecTest::DecodeABlock(bool lost_previous, bool lost_current) { + int16_t audio_type; + int value_1 = 0, value_2 = 0; + + if (lost_previous) { + // Decode previous frame. + if (!lost_current && + WebRtcOpus_PacketHasFec(&bit_stream_[0], encoded_bytes_) == 1) { + value_1 = + WebRtcOpus_DecodeFec(opus_decoder_, &bit_stream_[0], encoded_bytes_, + &out_data_[0], &audio_type); + } else { + // Call decoder PLC. + while (value_1 < static_cast(block_length_sample_)) { + int ret = WebRtcOpus_Decode(opus_decoder_, NULL, 0, &out_data_[value_1], + &audio_type); + EXPECT_EQ(ret, sampling_khz_ * 10); // Should return 10 ms of samples. + value_1 += ret; + } + } + EXPECT_EQ(static_cast(block_length_sample_), value_1); + } + + if (!lost_current) { + // Decode current frame. + value_2 = WebRtcOpus_Decode(opus_decoder_, &bit_stream_[0], encoded_bytes_, + &out_data_[value_1 * channels_], &audio_type); + EXPECT_EQ(static_cast(block_length_sample_), value_2); + } +} + +TEST_P(OpusFecTest, RandomPacketLossTest) { + const int kDurationMs = 200000; + int time_now_ms, fec_frames; + int actual_packet_loss_rate; + bool lost_current, lost_previous; + mode mode_set[3] = {{true, 0}, {false, 0}, {true, 50}}; + + lost_current = false; + for (int i = 0; i < 3; i++) { + if (mode_set[i].fec) { + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate( + opus_encoder_, mode_set[i].target_packet_loss_rate)); + printf("FEC is ON, target at packet loss rate %d percent.\n", + mode_set[i].target_packet_loss_rate); + } else { + EXPECT_EQ(0, WebRtcOpus_DisableFec(opus_encoder_)); + printf("FEC is OFF.\n"); + } + // In this test, we let the target packet loss rate match the actual rate. + actual_packet_loss_rate = mode_set[i].target_packet_loss_rate; + // Run every mode a certain time. + time_now_ms = 0; + fec_frames = 0; + while (time_now_ms < kDurationMs) { + // Encode & decode. + EncodeABlock(); + + // Check if payload has FEC. + int fec = WebRtcOpus_PacketHasFec(&bit_stream_[0], encoded_bytes_); + + // If FEC is disabled or the target packet loss rate is set to 0, there + // should be no FEC in the bit stream. + if (!mode_set[i].fec || mode_set[i].target_packet_loss_rate == 0) { + EXPECT_EQ(fec, 0); + } else if (fec == 1) { + fec_frames++; + } + + lost_previous = lost_current; + lost_current = rand() < actual_packet_loss_rate * (RAND_MAX / 100); + DecodeABlock(lost_previous, lost_current); + + time_now_ms += block_duration_ms_; + + // `data_pointer_` is incremented and wrapped across + // `loop_length_samples_`. + data_pointer_ = (data_pointer_ + block_length_sample_ * channels_) % + loop_length_samples_; + } + if (mode_set[i].fec) { + printf("%.2f percent frames has FEC.\n", + static_cast(fec_frames) * block_duration_ms_ / 2000); + } + } +} + +const coding_param param_set[] = { + std::make_tuple(1, + 64000, + string("audio_coding/testfile32kHz"), + string("pcm")), + std::make_tuple(1, + 32000, + string("audio_coding/testfile32kHz"), + string("pcm")), + std::make_tuple(2, + 64000, + string("audio_coding/teststereo32kHz"), + string("pcm"))}; + +// 64 kbps, stereo +INSTANTIATE_TEST_SUITE_P(AllTest, OpusFecTest, ::testing::ValuesIn(param_set)); + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_inst.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_inst.h new file mode 100644 index 0000000000..92c5c354a7 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_inst.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INST_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INST_H_ + +#include + +#include "rtc_base/ignore_wundef.h" + +RTC_PUSH_IGNORING_WUNDEF() +#include "third_party/opus/src/include/opus.h" +#include "third_party/opus/src/include/opus_multistream.h" +RTC_POP_IGNORING_WUNDEF() + +struct WebRtcOpusEncInst { + OpusEncoder* encoder; + OpusMSEncoder* multistream_encoder; + size_t channels; + int in_dtx_mode; + bool avoid_noise_pumping_during_dtx; + int sample_rate_hz; + float smooth_energy_non_active_frames; +}; + +struct WebRtcOpusDecInst { + OpusDecoder* decoder; + OpusMSDecoder* multistream_decoder; + int prev_decoded_samples; + bool plc_use_prev_decoded_samples; + size_t channels; + int in_dtx_mode; + int sample_rate_hz; +}; + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.cc new file mode 100644 index 0000000000..67d8619b34 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.cc @@ -0,0 +1,881 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/opus_interface.h" + +#include + +#include + +#include "api/array_view.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/field_trial.h" + +enum { +#if WEBRTC_OPUS_SUPPORT_120MS_PTIME + /* Maximum supported frame size in WebRTC is 120 ms. */ + kWebRtcOpusMaxEncodeFrameSizeMs = 120, +#else + /* Maximum supported frame size in WebRTC is 60 ms. */ + kWebRtcOpusMaxEncodeFrameSizeMs = 60, +#endif + + /* The format allows up to 120 ms frames. Since we don't control the other + * side, we must allow for packets of that size. NetEq is currently limited + * to 60 ms on the receive side. */ + kWebRtcOpusMaxDecodeFrameSizeMs = 120, + + // Duration of audio that each call to packet loss concealment covers. + kWebRtcOpusPlcFrameSizeMs = 10, +}; + +constexpr char kPlcUsePrevDecodedSamplesFieldTrial[] = + "WebRTC-Audio-OpusPlcUsePrevDecodedSamples"; + +constexpr char kAvoidNoisePumpingDuringDtxFieldTrial[] = + "WebRTC-Audio-OpusAvoidNoisePumpingDuringDtx"; + +constexpr char kSetSignalVoiceWithDtxFieldTrial[] = + "WebRTC-Audio-OpusSetSignalVoiceWithDtx"; + +static int FrameSizePerChannel(int frame_size_ms, int sample_rate_hz) { + RTC_DCHECK_GT(frame_size_ms, 0); + RTC_DCHECK_EQ(frame_size_ms % 10, 0); + RTC_DCHECK_GT(sample_rate_hz, 0); + RTC_DCHECK_EQ(sample_rate_hz % 1000, 0); + return frame_size_ms * (sample_rate_hz / 1000); +} + +// Maximum sample count per channel. +static int MaxFrameSizePerChannel(int sample_rate_hz) { + return FrameSizePerChannel(kWebRtcOpusMaxDecodeFrameSizeMs, sample_rate_hz); +} + +// Default sample count per channel. +static int DefaultFrameSizePerChannel(int sample_rate_hz) { + return FrameSizePerChannel(20, sample_rate_hz); +} + +// Returns true if the `encoded` payload corresponds to a refresh DTX packet +// whose energy is larger than the expected for non activity packets. +static bool WebRtcOpus_IsHighEnergyRefreshDtxPacket( + OpusEncInst* inst, + rtc::ArrayView frame, + rtc::ArrayView encoded) { + if (encoded.size() <= 2) { + return false; + } + int number_frames = + frame.size() / DefaultFrameSizePerChannel(inst->sample_rate_hz); + if (number_frames > 0 && + WebRtcOpus_PacketHasVoiceActivity(encoded.data(), encoded.size()) == 0) { + const float average_frame_energy = + std::accumulate(frame.begin(), frame.end(), 0.0f, + [](float a, int32_t b) { return a + b * b; }) / + number_frames; + if (WebRtcOpus_GetInDtx(inst) == 1 && + average_frame_energy >= inst->smooth_energy_non_active_frames * 0.5f) { + // This is a refresh DTX packet as the encoder is in DTX and has + // produced a payload > 2 bytes. This refresh packet has a higher energy + // than the smooth energy of non activity frames (with a 3 dB negative + // margin) and, therefore, it is flagged as a high energy refresh DTX + // packet. + return true; + } + // The average energy is tracked in a similar way as the modeling of the + // comfort noise in the Silk decoder in Opus + // (third_party/opus/src/silk/CNG.c). + if (average_frame_energy < inst->smooth_energy_non_active_frames * 0.5f) { + inst->smooth_energy_non_active_frames = average_frame_energy; + } else { + inst->smooth_energy_non_active_frames += + (average_frame_energy - inst->smooth_energy_non_active_frames) * + 0.25f; + } + } + return false; +} + +int16_t WebRtcOpus_EncoderCreate(OpusEncInst** inst, + size_t channels, + int32_t application, + int sample_rate_hz) { + int opus_app; + if (!inst) + return -1; + + switch (application) { + case 0: + opus_app = OPUS_APPLICATION_VOIP; + break; + case 1: + opus_app = OPUS_APPLICATION_AUDIO; + break; + default: + return -1; + } + + OpusEncInst* state = + reinterpret_cast(calloc(1, sizeof(OpusEncInst))); + RTC_DCHECK(state); + + int error; + state->encoder = opus_encoder_create( + sample_rate_hz, static_cast(channels), opus_app, &error); + + if (error != OPUS_OK || (!state->encoder && !state->multistream_encoder)) { + WebRtcOpus_EncoderFree(state); + return -1; + } + + state->in_dtx_mode = 0; + state->channels = channels; + state->sample_rate_hz = sample_rate_hz; + state->smooth_energy_non_active_frames = 0.0f; + state->avoid_noise_pumping_during_dtx = + webrtc::field_trial::IsEnabled(kAvoidNoisePumpingDuringDtxFieldTrial); + + *inst = state; + return 0; +} + +int16_t WebRtcOpus_MultistreamEncoderCreate( + OpusEncInst** inst, + size_t channels, + int32_t application, + size_t streams, + size_t coupled_streams, + const unsigned char* channel_mapping) { + int opus_app; + if (!inst) + return -1; + + switch (application) { + case 0: + opus_app = OPUS_APPLICATION_VOIP; + break; + case 1: + opus_app = OPUS_APPLICATION_AUDIO; + break; + default: + return -1; + } + + OpusEncInst* state = + reinterpret_cast(calloc(1, sizeof(OpusEncInst))); + RTC_DCHECK(state); + + int error; + const int sample_rate_hz = 48000; + state->multistream_encoder = opus_multistream_encoder_create( + sample_rate_hz, channels, streams, coupled_streams, channel_mapping, + opus_app, &error); + + if (error != OPUS_OK || (!state->encoder && !state->multistream_encoder)) { + WebRtcOpus_EncoderFree(state); + return -1; + } + + state->in_dtx_mode = 0; + state->channels = channels; + state->sample_rate_hz = sample_rate_hz; + state->smooth_energy_non_active_frames = 0.0f; + state->avoid_noise_pumping_during_dtx = false; + + *inst = state; + return 0; +} + +int16_t WebRtcOpus_EncoderFree(OpusEncInst* inst) { + if (inst) { + if (inst->encoder) { + opus_encoder_destroy(inst->encoder); + } else { + opus_multistream_encoder_destroy(inst->multistream_encoder); + } + free(inst); + return 0; + } else { + return -1; + } +} + +int WebRtcOpus_Encode(OpusEncInst* inst, + const int16_t* audio_in, + size_t samples, + size_t length_encoded_buffer, + uint8_t* encoded) { + int res; + + if (samples > 48 * kWebRtcOpusMaxEncodeFrameSizeMs) { + return -1; + } + + if (inst->encoder) { + res = opus_encode(inst->encoder, (const opus_int16*)audio_in, + static_cast(samples), encoded, + static_cast(length_encoded_buffer)); + } else { + res = opus_multistream_encode( + inst->multistream_encoder, (const opus_int16*)audio_in, + static_cast(samples), encoded, + static_cast(length_encoded_buffer)); + } + + if (res <= 0) { + return -1; + } + + if (res <= 2) { + // Indicates DTX since the packet has nothing but a header. In principle, + // there is no need to send this packet. However, we do transmit the first + // occurrence to let the decoder know that the encoder enters DTX mode. + if (inst->in_dtx_mode) { + return 0; + } else { + inst->in_dtx_mode = 1; + return res; + } + } + + if (inst->avoid_noise_pumping_during_dtx && WebRtcOpus_GetUseDtx(inst) == 1 && + WebRtcOpus_IsHighEnergyRefreshDtxPacket( + inst, rtc::MakeArrayView(audio_in, samples), + rtc::MakeArrayView(encoded, res))) { + // This packet is a high energy refresh DTX packet. For avoiding an increase + // of the energy in the DTX region at the decoder, this packet is + // substituted by a TOC byte with one empty frame. + // The number of frames described in the TOC byte + // (https://tools.ietf.org/html/rfc6716#section-3.1) are overwritten to + // always indicate one frame (last two bits equal to 0). + encoded[0] = encoded[0] & 0b11111100; + inst->in_dtx_mode = 1; + // The payload is just the TOC byte and has 1 byte as length. + return 1; + } + inst->in_dtx_mode = 0; + return res; +} + +#define ENCODER_CTL(inst, vargs) \ + (inst->encoder \ + ? opus_encoder_ctl(inst->encoder, vargs) \ + : opus_multistream_encoder_ctl(inst->multistream_encoder, vargs)) + +int16_t WebRtcOpus_SetBitRate(OpusEncInst* inst, int32_t rate) { + if (inst) { + return ENCODER_CTL(inst, OPUS_SET_BITRATE(rate)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetPacketLossRate(OpusEncInst* inst, int32_t loss_rate) { + if (inst) { + return ENCODER_CTL(inst, OPUS_SET_PACKET_LOSS_PERC(loss_rate)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetMaxPlaybackRate(OpusEncInst* inst, int32_t frequency_hz) { + opus_int32 set_bandwidth; + + if (!inst) + return -1; + + if (frequency_hz <= 8000) { + set_bandwidth = OPUS_BANDWIDTH_NARROWBAND; + } else if (frequency_hz <= 12000) { + set_bandwidth = OPUS_BANDWIDTH_MEDIUMBAND; + } else if (frequency_hz <= 16000) { + set_bandwidth = OPUS_BANDWIDTH_WIDEBAND; + } else if (frequency_hz <= 24000) { + set_bandwidth = OPUS_BANDWIDTH_SUPERWIDEBAND; + } else { + set_bandwidth = OPUS_BANDWIDTH_FULLBAND; + } + return ENCODER_CTL(inst, OPUS_SET_MAX_BANDWIDTH(set_bandwidth)); +} + +int16_t WebRtcOpus_GetMaxPlaybackRate(OpusEncInst* const inst, + int32_t* result_hz) { + if (inst->encoder) { + if (opus_encoder_ctl(inst->encoder, OPUS_GET_MAX_BANDWIDTH(result_hz)) == + OPUS_OK) { + return 0; + } + return -1; + } + + opus_int32 max_bandwidth; + int s; + int ret; + + max_bandwidth = 0; + ret = OPUS_OK; + s = 0; + while (ret == OPUS_OK) { + OpusEncoder* enc; + opus_int32 bandwidth; + + ret = ENCODER_CTL(inst, OPUS_MULTISTREAM_GET_ENCODER_STATE(s, &enc)); + if (ret == OPUS_BAD_ARG) + break; + if (ret != OPUS_OK) + return -1; + if (opus_encoder_ctl(enc, OPUS_GET_MAX_BANDWIDTH(&bandwidth)) != OPUS_OK) + return -1; + + if (max_bandwidth != 0 && max_bandwidth != bandwidth) + return -1; + + max_bandwidth = bandwidth; + s++; + } + *result_hz = max_bandwidth; + return 0; +} + +int16_t WebRtcOpus_EnableFec(OpusEncInst* inst) { + if (inst) { + return ENCODER_CTL(inst, OPUS_SET_INBAND_FEC(1)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_DisableFec(OpusEncInst* inst) { + if (inst) { + return ENCODER_CTL(inst, OPUS_SET_INBAND_FEC(0)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_EnableDtx(OpusEncInst* inst) { + if (inst) { + if (webrtc::field_trial::IsEnabled(kSetSignalVoiceWithDtxFieldTrial)) { + int ret = ENCODER_CTL(inst, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); + if (ret != OPUS_OK) { + return ret; + } + } + return ENCODER_CTL(inst, OPUS_SET_DTX(1)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_DisableDtx(OpusEncInst* inst) { + if (inst) { + if (webrtc::field_trial::IsEnabled(kSetSignalVoiceWithDtxFieldTrial)) { + int ret = ENCODER_CTL(inst, OPUS_SET_SIGNAL(OPUS_AUTO)); + if (ret != OPUS_OK) { + return ret; + } + } + return ENCODER_CTL(inst, OPUS_SET_DTX(0)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_GetUseDtx(OpusEncInst* inst) { + if (inst) { + opus_int32 use_dtx; + if (ENCODER_CTL(inst, OPUS_GET_DTX(&use_dtx)) == 0) { + return use_dtx; + } + } + return -1; +} + +int16_t WebRtcOpus_EnableCbr(OpusEncInst* inst) { + if (inst) { + return ENCODER_CTL(inst, OPUS_SET_VBR(0)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_DisableCbr(OpusEncInst* inst) { + if (inst) { + return ENCODER_CTL(inst, OPUS_SET_VBR(1)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetComplexity(OpusEncInst* inst, int32_t complexity) { + if (inst) { + return ENCODER_CTL(inst, OPUS_SET_COMPLEXITY(complexity)); + } else { + return -1; + } +} + +int32_t WebRtcOpus_GetBandwidth(OpusEncInst* inst) { + if (!inst) { + return -1; + } + int32_t bandwidth; + if (ENCODER_CTL(inst, OPUS_GET_BANDWIDTH(&bandwidth)) == 0) { + return bandwidth; + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetBandwidth(OpusEncInst* inst, int32_t bandwidth) { + if (inst) { + return ENCODER_CTL(inst, OPUS_SET_BANDWIDTH(bandwidth)); + } else { + return -1; + } +} + +int16_t WebRtcOpus_SetForceChannels(OpusEncInst* inst, size_t num_channels) { + if (!inst) + return -1; + if (num_channels == 0) { + return ENCODER_CTL(inst, OPUS_SET_FORCE_CHANNELS(OPUS_AUTO)); + } else if (num_channels == 1 || num_channels == 2) { + return ENCODER_CTL(inst, OPUS_SET_FORCE_CHANNELS(num_channels)); + } else { + return -1; + } +} + +int32_t WebRtcOpus_GetInDtx(OpusEncInst* inst) { + if (!inst) { + return -1; + } +#ifdef OPUS_GET_IN_DTX + int32_t in_dtx; + if (ENCODER_CTL(inst, OPUS_GET_IN_DTX(&in_dtx)) == 0) { + return in_dtx; + } +#endif + return -1; +} + +int16_t WebRtcOpus_DecoderCreate(OpusDecInst** inst, + size_t channels, + int sample_rate_hz) { + int error; + OpusDecInst* state; + + if (inst != NULL) { + // Create Opus decoder state. + state = reinterpret_cast(calloc(1, sizeof(OpusDecInst))); + if (state == NULL) { + return -1; + } + + state->decoder = + opus_decoder_create(sample_rate_hz, static_cast(channels), &error); + if (error == OPUS_OK && state->decoder) { + // Creation of memory all ok. + state->channels = channels; + state->sample_rate_hz = sample_rate_hz; + state->plc_use_prev_decoded_samples = + webrtc::field_trial::IsEnabled(kPlcUsePrevDecodedSamplesFieldTrial); + if (state->plc_use_prev_decoded_samples) { + state->prev_decoded_samples = + DefaultFrameSizePerChannel(state->sample_rate_hz); + } + state->in_dtx_mode = 0; + *inst = state; + return 0; + } + + // If memory allocation was unsuccessful, free the entire state. + if (state->decoder) { + opus_decoder_destroy(state->decoder); + } + free(state); + } + return -1; +} + +int16_t WebRtcOpus_MultistreamDecoderCreate( + OpusDecInst** inst, + size_t channels, + size_t streams, + size_t coupled_streams, + const unsigned char* channel_mapping) { + int error; + OpusDecInst* state; + + if (inst != NULL) { + // Create Opus decoder state. + state = reinterpret_cast(calloc(1, sizeof(OpusDecInst))); + if (state == NULL) { + return -1; + } + + // Create new memory, always at 48000 Hz. + state->multistream_decoder = opus_multistream_decoder_create( + 48000, channels, streams, coupled_streams, channel_mapping, &error); + + if (error == OPUS_OK && state->multistream_decoder) { + // Creation of memory all ok. + state->channels = channels; + state->sample_rate_hz = 48000; + state->plc_use_prev_decoded_samples = + webrtc::field_trial::IsEnabled(kPlcUsePrevDecodedSamplesFieldTrial); + if (state->plc_use_prev_decoded_samples) { + state->prev_decoded_samples = + DefaultFrameSizePerChannel(state->sample_rate_hz); + } + state->in_dtx_mode = 0; + *inst = state; + return 0; + } + + // If memory allocation was unsuccessful, free the entire state. + opus_multistream_decoder_destroy(state->multistream_decoder); + free(state); + } + return -1; +} + +int16_t WebRtcOpus_DecoderFree(OpusDecInst* inst) { + if (inst) { + if (inst->decoder) { + opus_decoder_destroy(inst->decoder); + } else if (inst->multistream_decoder) { + opus_multistream_decoder_destroy(inst->multistream_decoder); + } + free(inst); + return 0; + } else { + return -1; + } +} + +size_t WebRtcOpus_DecoderChannels(OpusDecInst* inst) { + return inst->channels; +} + +void WebRtcOpus_DecoderInit(OpusDecInst* inst) { + if (inst->decoder) { + opus_decoder_ctl(inst->decoder, OPUS_RESET_STATE); + } else { + opus_multistream_decoder_ctl(inst->multistream_decoder, OPUS_RESET_STATE); + } + inst->in_dtx_mode = 0; +} + +/* For decoder to determine if it is to output speech or comfort noise. */ +static int16_t DetermineAudioType(OpusDecInst* inst, size_t encoded_bytes) { + // Audio type becomes comfort noise if `encoded_byte` is 1 and keeps + // to be so if the following `encoded_byte` are 0 or 1. + if (encoded_bytes == 0 && inst->in_dtx_mode) { + return 2; // Comfort noise. + } else if (encoded_bytes == 1 || encoded_bytes == 2) { + // TODO(henrik.lundin): There is a slight risk that a 2-byte payload is in + // fact a 1-byte TOC with a 1-byte payload. That will be erroneously + // interpreted as comfort noise output, but such a payload is probably + // faulty anyway. + + // TODO(webrtc:10218): This is wrong for multistream opus. Then are several + // single-stream packets glued together with some packet size bytes in + // between. See https://tools.ietf.org/html/rfc6716#appendix-B + inst->in_dtx_mode = 1; + return 2; // Comfort noise. + } else { + inst->in_dtx_mode = 0; + return 0; // Speech. + } +} + +/* `frame_size` is set to maximum Opus frame size in the normal case, and + * is set to the number of samples needed for PLC in case of losses. + * It is up to the caller to make sure the value is correct. */ +static int DecodeNative(OpusDecInst* inst, + const uint8_t* encoded, + size_t encoded_bytes, + int frame_size, + int16_t* decoded, + int16_t* audio_type, + int decode_fec) { + int res = -1; + if (inst->decoder) { + res = opus_decode( + inst->decoder, encoded, static_cast(encoded_bytes), + reinterpret_cast(decoded), frame_size, decode_fec); + } else { + res = opus_multistream_decode(inst->multistream_decoder, encoded, + static_cast(encoded_bytes), + reinterpret_cast(decoded), + frame_size, decode_fec); + } + + if (res <= 0) + return -1; + + *audio_type = DetermineAudioType(inst, encoded_bytes); + + return res; +} + +static int DecodePlc(OpusDecInst* inst, int16_t* decoded) { + int16_t audio_type = 0; + int decoded_samples; + int plc_samples = + FrameSizePerChannel(kWebRtcOpusPlcFrameSizeMs, inst->sample_rate_hz); + + if (inst->plc_use_prev_decoded_samples) { + /* The number of samples we ask for is `number_of_lost_frames` times + * `prev_decoded_samples_`. Limit the number of samples to maximum + * `MaxFrameSizePerChannel()`. */ + plc_samples = inst->prev_decoded_samples; + const int max_samples_per_channel = + MaxFrameSizePerChannel(inst->sample_rate_hz); + plc_samples = plc_samples <= max_samples_per_channel + ? plc_samples + : max_samples_per_channel; + } + decoded_samples = + DecodeNative(inst, NULL, 0, plc_samples, decoded, &audio_type, 0); + if (decoded_samples < 0) { + return -1; + } + + return decoded_samples; +} + +int WebRtcOpus_Decode(OpusDecInst* inst, + const uint8_t* encoded, + size_t encoded_bytes, + int16_t* decoded, + int16_t* audio_type) { + int decoded_samples; + + if (encoded_bytes == 0) { + *audio_type = DetermineAudioType(inst, encoded_bytes); + decoded_samples = DecodePlc(inst, decoded); + } else { + decoded_samples = DecodeNative(inst, encoded, encoded_bytes, + MaxFrameSizePerChannel(inst->sample_rate_hz), + decoded, audio_type, 0); + } + if (decoded_samples < 0) { + return -1; + } + + if (inst->plc_use_prev_decoded_samples) { + /* Update decoded sample memory, to be used by the PLC in case of losses. */ + inst->prev_decoded_samples = decoded_samples; + } + + return decoded_samples; +} + +int WebRtcOpus_DecodeFec(OpusDecInst* inst, + const uint8_t* encoded, + size_t encoded_bytes, + int16_t* decoded, + int16_t* audio_type) { + int decoded_samples; + int fec_samples; + + if (WebRtcOpus_PacketHasFec(encoded, encoded_bytes) != 1) { + return 0; + } + + fec_samples = + opus_packet_get_samples_per_frame(encoded, inst->sample_rate_hz); + + decoded_samples = DecodeNative(inst, encoded, encoded_bytes, fec_samples, + decoded, audio_type, 1); + if (decoded_samples < 0) { + return -1; + } + + return decoded_samples; +} + +int WebRtcOpus_DurationEst(OpusDecInst* inst, + const uint8_t* payload, + size_t payload_length_bytes) { + if (payload_length_bytes == 0) { + // WebRtcOpus_Decode calls PLC when payload length is zero. So we return + // PLC duration correspondingly. + return WebRtcOpus_PlcDuration(inst); + } + + int frames, samples; + frames = opus_packet_get_nb_frames( + payload, static_cast(payload_length_bytes)); + if (frames < 0) { + /* Invalid payload data. */ + return 0; + } + samples = + frames * opus_packet_get_samples_per_frame(payload, inst->sample_rate_hz); + if (samples > 120 * inst->sample_rate_hz / 1000) { + // More than 120 ms' worth of samples. + return 0; + } + return samples; +} + +int WebRtcOpus_PlcDuration(OpusDecInst* inst) { + if (inst->plc_use_prev_decoded_samples) { + /* The number of samples we ask for is `number_of_lost_frames` times + * `prev_decoded_samples_`. Limit the number of samples to maximum + * `MaxFrameSizePerChannel()`. */ + const int plc_samples = inst->prev_decoded_samples; + const int max_samples_per_channel = + MaxFrameSizePerChannel(inst->sample_rate_hz); + return plc_samples <= max_samples_per_channel ? plc_samples + : max_samples_per_channel; + } + return FrameSizePerChannel(kWebRtcOpusPlcFrameSizeMs, inst->sample_rate_hz); +} + +int WebRtcOpus_FecDurationEst(const uint8_t* payload, + size_t payload_length_bytes, + int sample_rate_hz) { + if (WebRtcOpus_PacketHasFec(payload, payload_length_bytes) != 1) { + return 0; + } + const int samples = + opus_packet_get_samples_per_frame(payload, sample_rate_hz); + const int samples_per_ms = sample_rate_hz / 1000; + if (samples < 10 * samples_per_ms || samples > 120 * samples_per_ms) { + /* Invalid payload duration. */ + return 0; + } + return samples; +} + +int WebRtcOpus_NumSilkFrames(const uint8_t* payload) { + // For computing the payload length in ms, the sample rate is not important + // since it cancels out. We use 48 kHz, but any valid sample rate would work. + int payload_length_ms = + opus_packet_get_samples_per_frame(payload, 48000) / 48; + if (payload_length_ms < 10) + payload_length_ms = 10; + + int silk_frames; + switch (payload_length_ms) { + case 10: + case 20: + silk_frames = 1; + break; + case 40: + silk_frames = 2; + break; + case 60: + silk_frames = 3; + break; + default: + return 0; // It is actually even an invalid packet. + } + return silk_frames; +} + +// This method is based on Definition of the Opus Audio Codec +// (https://tools.ietf.org/html/rfc6716). Basically, this method is based on +// parsing the LP layer of an Opus packet, particularly the LBRR flag. +int WebRtcOpus_PacketHasFec(const uint8_t* payload, + size_t payload_length_bytes) { + if (payload == NULL || payload_length_bytes == 0) + return 0; + + // In CELT_ONLY mode, packets should not have FEC. + if (payload[0] & 0x80) + return 0; + + int silk_frames = WebRtcOpus_NumSilkFrames(payload); + if (silk_frames == 0) + return 0; // Not valid. + + const int channels = opus_packet_get_nb_channels(payload); + RTC_DCHECK(channels == 1 || channels == 2); + + // Max number of frames in an Opus packet is 48. + opus_int16 frame_sizes[48]; + const unsigned char* frame_data[48]; + + // Parse packet to get the frames. But we only care about the first frame, + // since we can only decode the FEC from the first one. + if (opus_packet_parse(payload, static_cast(payload_length_bytes), + NULL, frame_data, frame_sizes, NULL) < 0) { + return 0; + } + + if (frame_sizes[0] < 1) { + return 0; + } + + // A frame starts with the LP layer. The LP layer begins with two to eight + // header bits.These consist of one VAD bit per SILK frame (up to 3), + // followed by a single flag indicating the presence of LBRR frames. + // For a stereo packet, these first flags correspond to the mid channel, and + // a second set of flags is included for the side channel. Because these are + // the first symbols decoded by the range coder and because they are coded + // as binary values with uniform probability, they can be extracted directly + // from the most significant bits of the first byte of compressed data. + for (int n = 0; n < channels; n++) { + // The LBRR bit for channel 1 is on the (`silk_frames` + 1)-th bit, and + // that of channel 2 is on the |(`silk_frames` + 1) * 2 + 1|-th bit. + if (frame_data[0][0] & (0x80 >> ((n + 1) * (silk_frames + 1) - 1))) + return 1; + } + + return 0; +} + +int WebRtcOpus_PacketHasVoiceActivity(const uint8_t* payload, + size_t payload_length_bytes) { + if (payload == NULL || payload_length_bytes == 0) + return 0; + + // In CELT_ONLY mode we can not determine whether there is VAD. + if (payload[0] & 0x80) + return -1; + + int silk_frames = WebRtcOpus_NumSilkFrames(payload); + if (silk_frames == 0) + return -1; + + const int channels = opus_packet_get_nb_channels(payload); + RTC_DCHECK(channels == 1 || channels == 2); + + // Max number of frames in an Opus packet is 48. + opus_int16 frame_sizes[48]; + const unsigned char* frame_data[48]; + + // Parse packet to get the frames. + int frames = + opus_packet_parse(payload, static_cast(payload_length_bytes), + NULL, frame_data, frame_sizes, NULL); + if (frames < 0) + return -1; + + // Iterate over all Opus frames which may contain multiple SILK frames. + for (int frame = 0; frame < frames; frame++) { + if (frame_sizes[frame] < 1) { + continue; + } + if (frame_data[frame][0] >> (8 - silk_frames)) + return 1; + if (channels == 2 && + (frame_data[frame][0] << (silk_frames + 1)) >> (8 - silk_frames)) + return 1; + } + + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.h new file mode 100644 index 0000000000..89159ce1c0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.h @@ -0,0 +1,547 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INTERFACE_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INTERFACE_H_ + +#include +#include + +#include "modules/audio_coding/codecs/opus/opus_inst.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Opaque wrapper types for the codec state. +typedef struct WebRtcOpusEncInst OpusEncInst; +typedef struct WebRtcOpusDecInst OpusDecInst; + +/**************************************************************************** + * WebRtcOpus_EncoderCreate(...) + * + * This function creates an Opus encoder that encodes mono or stereo. + * + * Input: + * - channels : number of channels; 1 or 2. + * - application : 0 - VOIP applications. + * Favor speech intelligibility. + * 1 - Audio applications. + * Favor faithfulness to the original input. + * - sample_rate_hz : sample rate of input audio + * + * Output: + * - inst : a pointer to Encoder context that is created + * if success. + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EncoderCreate(OpusEncInst** inst, + size_t channels, + int32_t application, + int sample_rate_hz); + +/**************************************************************************** + * WebRtcOpus_MultistreamEncoderCreate(...) + * + * This function creates an Opus encoder with any supported channel count. + * + * Input: + * - channels : number of channels in the input of the encoder. + * - application : 0 - VOIP applications. + * Favor speech intelligibility. + * 1 - Audio applications. + * Favor faithfulness to the original input. + * - streams : number of streams, as described in RFC 7845. + * - coupled_streams : number of coupled streams, as described in + * RFC 7845. + * - channel_mapping : the channel mapping; pointer to array of + * `channel` bytes, as described in RFC 7845. + * + * Output: + * - inst : a pointer to Encoder context that is created + * if success. + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_MultistreamEncoderCreate( + OpusEncInst** inst, + size_t channels, + int32_t application, + size_t streams, + size_t coupled_streams, + const unsigned char* channel_mapping); + +int16_t WebRtcOpus_EncoderFree(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_Encode(...) + * + * This function encodes audio as a series of Opus frames and inserts + * it into a packet. Input buffer can be any length. + * + * Input: + * - inst : Encoder context + * - audio_in : Input speech data buffer + * - samples : Samples per channel in audio_in + * - length_encoded_buffer : Output buffer size + * + * Output: + * - encoded : Output compressed data buffer + * + * Return value : >=0 - Length (in bytes) of coded data + * -1 - Error + */ +int WebRtcOpus_Encode(OpusEncInst* inst, + const int16_t* audio_in, + size_t samples, + size_t length_encoded_buffer, + uint8_t* encoded); + +/**************************************************************************** + * WebRtcOpus_SetBitRate(...) + * + * This function adjusts the target bitrate of the encoder. + * + * Input: + * - inst : Encoder context + * - rate : New target bitrate + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetBitRate(OpusEncInst* inst, int32_t rate); + +/**************************************************************************** + * WebRtcOpus_SetPacketLossRate(...) + * + * This function configures the encoder's expected packet loss percentage. + * + * Input: + * - inst : Encoder context + * - loss_rate : loss percentage in the range 0-100, inclusive. + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetPacketLossRate(OpusEncInst* inst, int32_t loss_rate); + +/**************************************************************************** + * WebRtcOpus_SetMaxPlaybackRate(...) + * + * Configures the maximum playback rate for encoding. Due to hardware + * limitations, the receiver may render audio up to a playback rate. Opus + * encoder can use this information to optimize for network usage and encoding + * complexity. This will affect the audio bandwidth in the coded audio. However, + * the input/output sample rate is not affected. + * + * Input: + * - inst : Encoder context + * - frequency_hz : Maximum playback rate in Hz. + * This parameter can take any value. The relation + * between the value and the Opus internal mode is + * as following: + * frequency_hz <= 8000 narrow band + * 8000 < frequency_hz <= 12000 medium band + * 12000 < frequency_hz <= 16000 wide band + * 16000 < frequency_hz <= 24000 super wide band + * frequency_hz > 24000 full band + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetMaxPlaybackRate(OpusEncInst* inst, int32_t frequency_hz); + +/**************************************************************************** + * WebRtcOpus_GetMaxPlaybackRate(...) + * + * Queries the maximum playback rate for encoding. If different single-stream + * encoders have different maximum playback rates, this function fails. + * + * Input: + * - inst : Encoder context. + * Output: + * - result_hz : The maximum playback rate in Hz. + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_GetMaxPlaybackRate(OpusEncInst* const inst, + int32_t* result_hz); + +/* TODO(minyue): Check whether an API to check the FEC and the packet loss rate + * is needed. It might not be very useful since there are not many use cases and + * the caller can always maintain the states. */ + +/**************************************************************************** + * WebRtcOpus_EnableFec() + * + * This function enables FEC for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EnableFec(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_DisableFec() + * + * This function disables FEC for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_DisableFec(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_EnableDtx() + * + * This function enables Opus internal DTX for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EnableDtx(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_DisableDtx() + * + * This function disables Opus internal DTX for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_DisableDtx(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_GetUseDtx() + * + * This function gets the DTX configuration used for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Encoder does not use DTX. + * 1 - Encoder uses DTX. + * -1 - Error. + */ +int16_t WebRtcOpus_GetUseDtx(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_EnableCbr() + * + * This function enables CBR for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_EnableCbr(OpusEncInst* inst); + +/**************************************************************************** + * WebRtcOpus_DisableCbr() + * + * This function disables CBR for encoding. + * + * Input: + * - inst : Encoder context + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_DisableCbr(OpusEncInst* inst); + +/* + * WebRtcOpus_SetComplexity(...) + * + * This function adjusts the computational complexity. The effect is the same as + * calling the complexity setting of Opus as an Opus encoder related CTL. + * + * Input: + * - inst : Encoder context + * - complexity : New target complexity (0-10, inclusive) + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetComplexity(OpusEncInst* inst, int32_t complexity); + +/* + * WebRtcOpus_GetBandwidth(...) + * + * This function returns the current bandwidth. + * + * Input: + * - inst : Encoder context + * + * Return value : Bandwidth - Success + * -1 - Error + */ +int32_t WebRtcOpus_GetBandwidth(OpusEncInst* inst); + +/* + * WebRtcOpus_SetBandwidth(...) + * + * By default Opus decides which bandwidth to encode the signal in depending on + * the the bitrate. This function overrules the previous setting and forces the + * encoder to encode in narrowband/wideband/fullband/etc. + * + * Input: + * - inst : Encoder context + * - bandwidth : New target bandwidth. Valid values are: + * OPUS_BANDWIDTH_NARROWBAND + * OPUS_BANDWIDTH_MEDIUMBAND + * OPUS_BANDWIDTH_WIDEBAND + * OPUS_BANDWIDTH_SUPERWIDEBAND + * OPUS_BANDWIDTH_FULLBAND + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetBandwidth(OpusEncInst* inst, int32_t bandwidth); + +/* + * WebRtcOpus_GetInDtx(...) + * + * Gets the DTX state of the encoder. + * + * Input: + * - inst : Encoder context + * + * Return value : -1 - Error. + * 1 - Last encoded frame was comfort noise update during DTX. + * 0 - Last encoded frame was encoded with encoder not in DTX. + */ +int32_t WebRtcOpus_GetInDtx(OpusEncInst* inst); + +/* + * WebRtcOpus_SetForceChannels(...) + * + * If the encoder is initialized as a stereo encoder, Opus will by default + * decide whether to encode in mono or stereo based on the bitrate. This + * function overrules the previous setting, and forces the encoder to encode + * in auto/mono/stereo. + * + * If the Encoder is initialized as a mono encoder, and one tries to force + * stereo, the function will return an error. + * + * Input: + * - inst : Encoder context + * - num_channels : 0 - Not forced + * 1 - Mono + * 2 - Stereo + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_SetForceChannels(OpusEncInst* inst, size_t num_channels); + +int16_t WebRtcOpus_DecoderCreate(OpusDecInst** inst, + size_t channels, + int sample_rate_hz); + +/**************************************************************************** + * WebRtcOpus_MultistreamDecoderCreate(...) + * + * This function creates an Opus decoder with any supported channel count. + * + * Input: + * - channels : number of output channels that the decoder + * will produce. + * - streams : number of encoded streams, as described in + * RFC 7845. + * - coupled_streams : number of coupled streams, as described in + * RFC 7845. + * - channel_mapping : the channel mapping; pointer to array of + * `channel` bytes, as described in RFC 7845. + * + * Output: + * - inst : a pointer to a Decoder context that is created + * if success. + * + * Return value : 0 - Success + * -1 - Error + */ +int16_t WebRtcOpus_MultistreamDecoderCreate( + OpusDecInst** inst, + size_t channels, + size_t streams, + size_t coupled_streams, + const unsigned char* channel_mapping); + +int16_t WebRtcOpus_DecoderFree(OpusDecInst* inst); + +/**************************************************************************** + * WebRtcOpus_DecoderChannels(...) + * + * This function returns the number of channels created for Opus decoder. + */ +size_t WebRtcOpus_DecoderChannels(OpusDecInst* inst); + +/**************************************************************************** + * WebRtcOpus_DecoderInit(...) + * + * This function resets state of the decoder. + * + * Input: + * - inst : Decoder context + */ +void WebRtcOpus_DecoderInit(OpusDecInst* inst); + +/**************************************************************************** + * WebRtcOpus_Decode(...) + * + * This function decodes an Opus packet into one or more audio frames at the + * ACM interface's sampling rate (32 kHz). + * + * Input: + * - inst : Decoder context + * - encoded : Encoded data + * - encoded_bytes : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector + * - audio_type : 1 normal, 2 CNG (for Opus it should + * always return 1 since we're not using Opus's + * built-in DTX/CNG scheme) + * + * Return value : >0 - Samples per channel in decoded vector + * -1 - Error + */ +int WebRtcOpus_Decode(OpusDecInst* inst, + const uint8_t* encoded, + size_t encoded_bytes, + int16_t* decoded, + int16_t* audio_type); + +/**************************************************************************** + * WebRtcOpus_DecodeFec(...) + * + * This function decodes the FEC data from an Opus packet into one or more audio + * frames at the ACM interface's sampling rate (32 kHz). + * + * Input: + * - inst : Decoder context + * - encoded : Encoded data + * - encoded_bytes : Bytes in encoded vector + * + * Output: + * - decoded : The decoded vector (previous frame) + * + * Return value : >0 - Samples per channel in decoded vector + * 0 - No FEC data in the packet + * -1 - Error + */ +int WebRtcOpus_DecodeFec(OpusDecInst* inst, + const uint8_t* encoded, + size_t encoded_bytes, + int16_t* decoded, + int16_t* audio_type); + +/**************************************************************************** + * WebRtcOpus_DurationEst(...) + * + * This function calculates the duration of an opus packet. + * Input: + * - inst : Decoder context + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : The duration of the packet, in samples per + * channel. + */ +int WebRtcOpus_DurationEst(OpusDecInst* inst, + const uint8_t* payload, + size_t payload_length_bytes); + +/**************************************************************************** + * WebRtcOpus_PlcDuration(...) + * + * This function calculates the duration of a frame returned by packet loss + * concealment (PLC). + * + * Input: + * - inst : Decoder context + * + * Return value : The duration of a frame returned by PLC, in + * samples per channel. + */ +int WebRtcOpus_PlcDuration(OpusDecInst* inst); + +/* TODO(minyue): Check whether it is needed to add a decoder context to the + * arguments, like WebRtcOpus_DurationEst(...). In fact, the packet itself tells + * the duration. The decoder context in WebRtcOpus_DurationEst(...) is not used. + * So it may be advisable to remove it from WebRtcOpus_DurationEst(...). */ + +/**************************************************************************** + * WebRtcOpus_FecDurationEst(...) + * + * This function calculates the duration of the FEC data within an opus packet. + * Input: + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * - sample_rate_hz : Sample rate of output audio + * + * Return value : >0 - The duration of the FEC data in the + * packet in samples per channel. + * 0 - No FEC data in the packet. + */ +int WebRtcOpus_FecDurationEst(const uint8_t* payload, + size_t payload_length_bytes, + int sample_rate_hz); + +/**************************************************************************** + * WebRtcOpus_PacketHasFec(...) + * + * This function detects if an opus packet has FEC. + * Input: + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : 0 - the packet does NOT contain FEC. + * 1 - the packet contains FEC. + */ +int WebRtcOpus_PacketHasFec(const uint8_t* payload, + size_t payload_length_bytes); + +/**************************************************************************** + * WebRtcOpus_PacketHasVoiceActivity(...) + * + * This function returns the SILK VAD information encoded in the opus packet. + * For CELT-only packets that do not have VAD information, it returns -1. + * Input: + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : 0 - no frame had the VAD flag set. + * 1 - at least one frame had the VAD flag set. + * -1 - VAD status could not be determined. + */ +int WebRtcOpus_PacketHasVoiceActivity(const uint8_t* payload, + size_t payload_length_bytes); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_OPUS_INTERFACE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_speed_test.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_speed_test.cc new file mode 100644 index 0000000000..4477e8a5f8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_speed_test.cc @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/codecs/tools/audio_codec_speed_test.h" + +using ::std::string; + +namespace webrtc { + +static const int kOpusBlockDurationMs = 20; +static const int kOpusSamplingKhz = 48; + +class OpusSpeedTest : public AudioCodecSpeedTest { + protected: + OpusSpeedTest(); + void SetUp() override; + void TearDown() override; + float EncodeABlock(int16_t* in_data, + uint8_t* bit_stream, + size_t max_bytes, + size_t* encoded_bytes) override; + float DecodeABlock(const uint8_t* bit_stream, + size_t encoded_bytes, + int16_t* out_data) override; + WebRtcOpusEncInst* opus_encoder_; + WebRtcOpusDecInst* opus_decoder_; +}; + +OpusSpeedTest::OpusSpeedTest() + : AudioCodecSpeedTest(kOpusBlockDurationMs, + kOpusSamplingKhz, + kOpusSamplingKhz), + opus_encoder_(NULL), + opus_decoder_(NULL) {} + +void OpusSpeedTest::SetUp() { + AudioCodecSpeedTest::SetUp(); + // If channels_ == 1, use Opus VOIP mode, otherwise, audio mode. + int app = channels_ == 1 ? 0 : 1; + /* Create encoder memory. */ + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, app, 48000)); + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(&opus_decoder_, channels_, 48000)); + /* Set bitrate. */ + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, bit_rate_)); +} + +void OpusSpeedTest::TearDown() { + AudioCodecSpeedTest::TearDown(); + /* Free memory. */ + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +float OpusSpeedTest::EncodeABlock(int16_t* in_data, + uint8_t* bit_stream, + size_t max_bytes, + size_t* encoded_bytes) { + clock_t clocks = clock(); + int value = WebRtcOpus_Encode(opus_encoder_, in_data, input_length_sample_, + max_bytes, bit_stream); + clocks = clock() - clocks; + EXPECT_GT(value, 0); + *encoded_bytes = static_cast(value); + return 1000.0 * clocks / CLOCKS_PER_SEC; +} + +float OpusSpeedTest::DecodeABlock(const uint8_t* bit_stream, + size_t encoded_bytes, + int16_t* out_data) { + int value; + int16_t audio_type; + clock_t clocks = clock(); + value = WebRtcOpus_Decode(opus_decoder_, bit_stream, encoded_bytes, out_data, + &audio_type); + clocks = clock() - clocks; + EXPECT_EQ(output_length_sample_, static_cast(value)); + return 1000.0 * clocks / CLOCKS_PER_SEC; +} + +/* Test audio length in second. */ +constexpr size_t kDurationSec = 400; + +#define ADD_TEST(complexity) \ + TEST_P(OpusSpeedTest, OpusSetComplexityTest##complexity) { \ + /* Set complexity. */ \ + printf("Setting complexity to %d ...\n", complexity); \ + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_encoder_, complexity)); \ + EncodeDecode(kDurationSec); \ + } + +ADD_TEST(10) +ADD_TEST(9) +ADD_TEST(8) +ADD_TEST(7) +ADD_TEST(6) +ADD_TEST(5) +ADD_TEST(4) +ADD_TEST(3) +ADD_TEST(2) +ADD_TEST(1) +ADD_TEST(0) + +#define ADD_BANDWIDTH_TEST(bandwidth) \ + TEST_P(OpusSpeedTest, OpusSetBandwidthTest##bandwidth) { \ + /* Set bandwidth. */ \ + printf("Setting bandwidth to %d ...\n", bandwidth); \ + EXPECT_EQ(0, WebRtcOpus_SetBandwidth(opus_encoder_, bandwidth)); \ + EncodeDecode(kDurationSec); \ + } + +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_NARROWBAND) +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_MEDIUMBAND) +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_WIDEBAND) +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_SUPERWIDEBAND) +ADD_BANDWIDTH_TEST(OPUS_BANDWIDTH_FULLBAND) + +// List all test cases: (channel, bit rat, filename, extension). +const coding_param param_set[] = { + std::make_tuple(1, + 64000, + string("audio_coding/speech_mono_32_48kHz"), + string("pcm"), + true), + std::make_tuple(1, + 32000, + string("audio_coding/speech_mono_32_48kHz"), + string("pcm"), + true), + std::make_tuple(2, + 64000, + string("audio_coding/music_stereo_48kHz"), + string("pcm"), + true)}; + +INSTANTIATE_TEST_SUITE_P(AllTest, + OpusSpeedTest, + ::testing::ValuesIn(param_set)); + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_unittest.cc new file mode 100644 index 0000000000..4a9156ad58 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_unittest.cc @@ -0,0 +1,979 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "modules/audio_coding/codecs/opus/opus_inst.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +namespace { +// Equivalent to SDP params +// {{"channel_mapping", "0,1,2,3"}, {"coupled_streams", "2"}}. +constexpr unsigned char kQuadChannelMapping[] = {0, 1, 2, 3}; +constexpr int kQuadTotalStreams = 2; +constexpr int kQuadCoupledStreams = 2; + +constexpr unsigned char kStereoChannelMapping[] = {0, 1}; +constexpr int kStereoTotalStreams = 1; +constexpr int kStereoCoupledStreams = 1; + +constexpr unsigned char kMonoChannelMapping[] = {0}; +constexpr int kMonoTotalStreams = 1; +constexpr int kMonoCoupledStreams = 0; + +void CreateSingleOrMultiStreamEncoder(WebRtcOpusEncInst** opus_encoder, + int channels, + int application, + bool use_multistream, + int encoder_sample_rate_hz) { + EXPECT_TRUE(channels == 1 || channels == 2 || use_multistream); + if (use_multistream) { + EXPECT_EQ(encoder_sample_rate_hz, 48000); + if (channels == 1) { + EXPECT_EQ(0, WebRtcOpus_MultistreamEncoderCreate( + opus_encoder, channels, application, kMonoTotalStreams, + kMonoCoupledStreams, kMonoChannelMapping)); + } else if (channels == 2) { + EXPECT_EQ(0, WebRtcOpus_MultistreamEncoderCreate( + opus_encoder, channels, application, kStereoTotalStreams, + kStereoCoupledStreams, kStereoChannelMapping)); + } else if (channels == 4) { + EXPECT_EQ(0, WebRtcOpus_MultistreamEncoderCreate( + opus_encoder, channels, application, kQuadTotalStreams, + kQuadCoupledStreams, kQuadChannelMapping)); + } else { + EXPECT_TRUE(false) << channels; + } + } else { + EXPECT_EQ(0, WebRtcOpus_EncoderCreate(opus_encoder, channels, application, + encoder_sample_rate_hz)); + } +} + +void CreateSingleOrMultiStreamDecoder(WebRtcOpusDecInst** opus_decoder, + int channels, + bool use_multistream, + int decoder_sample_rate_hz) { + EXPECT_TRUE(channels == 1 || channels == 2 || use_multistream); + if (use_multistream) { + EXPECT_EQ(decoder_sample_rate_hz, 48000); + if (channels == 1) { + EXPECT_EQ(0, WebRtcOpus_MultistreamDecoderCreate( + opus_decoder, channels, kMonoTotalStreams, + kMonoCoupledStreams, kMonoChannelMapping)); + } else if (channels == 2) { + EXPECT_EQ(0, WebRtcOpus_MultistreamDecoderCreate( + opus_decoder, channels, kStereoTotalStreams, + kStereoCoupledStreams, kStereoChannelMapping)); + } else if (channels == 4) { + EXPECT_EQ(0, WebRtcOpus_MultistreamDecoderCreate( + opus_decoder, channels, kQuadTotalStreams, + kQuadCoupledStreams, kQuadChannelMapping)); + } else { + EXPECT_TRUE(false) << channels; + } + } else { + EXPECT_EQ(0, WebRtcOpus_DecoderCreate(opus_decoder, channels, + decoder_sample_rate_hz)); + } +} + +int SamplesPerChannel(int sample_rate_hz, int duration_ms) { + const int samples_per_ms = rtc::CheckedDivExact(sample_rate_hz, 1000); + return samples_per_ms * duration_ms; +} + +using test::AudioLoop; +using ::testing::Combine; +using ::testing::TestWithParam; +using ::testing::Values; + +// Maximum number of bytes in output bitstream. +const size_t kMaxBytes = 2000; + +class OpusTest + : public TestWithParam<::testing::tuple> { + protected: + OpusTest() = default; + + void TestDtxEffect(bool dtx, int block_length_ms); + + void TestCbrEffect(bool dtx, int block_length_ms); + + // Prepare `speech_data_` for encoding, read from a hard-coded file. + // After preparation, `speech_data_.GetNextBlock()` returns a pointer to a + // block of `block_length_ms` milliseconds. The data is looped every + // `loop_length_ms` milliseconds. + void PrepareSpeechData(int block_length_ms, int loop_length_ms); + + int EncodeDecode(WebRtcOpusEncInst* encoder, + rtc::ArrayView input_audio, + WebRtcOpusDecInst* decoder, + int16_t* output_audio, + int16_t* audio_type); + + void SetMaxPlaybackRate(WebRtcOpusEncInst* encoder, + opus_int32 expect, + int32_t set); + + void CheckAudioBounded(const int16_t* audio, + size_t samples, + size_t channels, + uint16_t bound) const; + + WebRtcOpusEncInst* opus_encoder_ = nullptr; + WebRtcOpusDecInst* opus_decoder_ = nullptr; + AudioLoop speech_data_; + uint8_t bitstream_[kMaxBytes]; + size_t encoded_bytes_ = 0; + const size_t channels_{std::get<0>(GetParam())}; + const int application_{std::get<1>(GetParam())}; + const bool use_multistream_{std::get<2>(GetParam())}; + const int encoder_sample_rate_hz_{std::get<3>(GetParam())}; + const int decoder_sample_rate_hz_{std::get<4>(GetParam())}; +}; + +} // namespace + +// Singlestream: Try all combinations. +INSTANTIATE_TEST_SUITE_P(Singlestream, + OpusTest, + testing::Combine(testing::Values(1, 2), + testing::Values(0, 1), + testing::Values(false), + testing::Values(16000, 48000), + testing::Values(16000, 48000))); + +// Multistream: Some representative cases (only 48 kHz for now). +INSTANTIATE_TEST_SUITE_P( + Multistream, + OpusTest, + testing::Values(std::make_tuple(1, 0, true, 48000, 48000), + std::make_tuple(2, 1, true, 48000, 48000), + std::make_tuple(4, 0, true, 48000, 48000), + std::make_tuple(4, 1, true, 48000, 48000))); + +void OpusTest::PrepareSpeechData(int block_length_ms, int loop_length_ms) { + std::map channel_to_basename = { + {1, "audio_coding/testfile32kHz"}, + {2, "audio_coding/teststereo32kHz"}, + {4, "audio_coding/speech_4_channels_48k_one_second"}}; + std::map channel_to_suffix = { + {1, "pcm"}, {2, "pcm"}, {4, "wav"}}; + const std::string file_name = webrtc::test::ResourcePath( + channel_to_basename[channels_], channel_to_suffix[channels_]); + if (loop_length_ms < block_length_ms) { + loop_length_ms = block_length_ms; + } + const int sample_rate_khz = + rtc::CheckedDivExact(encoder_sample_rate_hz_, 1000); + EXPECT_TRUE(speech_data_.Init(file_name, + loop_length_ms * sample_rate_khz * channels_, + block_length_ms * sample_rate_khz * channels_)); +} + +void OpusTest::SetMaxPlaybackRate(WebRtcOpusEncInst* encoder, + opus_int32 expect, + int32_t set) { + opus_int32 bandwidth; + EXPECT_EQ(0, WebRtcOpus_SetMaxPlaybackRate(opus_encoder_, set)); + EXPECT_EQ(0, WebRtcOpus_GetMaxPlaybackRate(opus_encoder_, &bandwidth)); + EXPECT_EQ(expect, bandwidth); +} + +void OpusTest::CheckAudioBounded(const int16_t* audio, + size_t samples, + size_t channels, + uint16_t bound) const { + for (size_t i = 0; i < samples; ++i) { + for (size_t c = 0; c < channels; ++c) { + ASSERT_GE(audio[i * channels + c], -bound); + ASSERT_LE(audio[i * channels + c], bound); + } + } +} + +int OpusTest::EncodeDecode(WebRtcOpusEncInst* encoder, + rtc::ArrayView input_audio, + WebRtcOpusDecInst* decoder, + int16_t* output_audio, + int16_t* audio_type) { + const int input_samples_per_channel = + rtc::CheckedDivExact(input_audio.size(), channels_); + int encoded_bytes_int = + WebRtcOpus_Encode(encoder, input_audio.data(), input_samples_per_channel, + kMaxBytes, bitstream_); + EXPECT_GE(encoded_bytes_int, 0); + encoded_bytes_ = static_cast(encoded_bytes_int); + if (encoded_bytes_ != 0) { + int est_len = WebRtcOpus_DurationEst(decoder, bitstream_, encoded_bytes_); + int act_len = WebRtcOpus_Decode(decoder, bitstream_, encoded_bytes_, + output_audio, audio_type); + EXPECT_EQ(est_len, act_len); + return act_len; + } else { + int total_dtx_len = 0; + const int output_samples_per_channel = input_samples_per_channel * + decoder_sample_rate_hz_ / + encoder_sample_rate_hz_; + while (total_dtx_len < output_samples_per_channel) { + int est_len = WebRtcOpus_DurationEst(decoder, NULL, 0); + int act_len = WebRtcOpus_Decode(decoder, NULL, 0, + &output_audio[total_dtx_len * channels_], + audio_type); + EXPECT_EQ(est_len, act_len); + total_dtx_len += act_len; + } + return total_dtx_len; + } +} + +// Test if encoder/decoder can enter DTX mode properly and do not enter DTX when +// they should not. This test is signal dependent. +void OpusTest::TestDtxEffect(bool dtx, int block_length_ms) { + PrepareSpeechData(block_length_ms, 2000); + const size_t input_samples = + rtc::CheckedDivExact(encoder_sample_rate_hz_, 1000) * block_length_ms; + const size_t output_samples = + rtc::CheckedDivExact(decoder_sample_rate_hz_, 1000) * block_length_ms; + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + + // Set bitrate. + EXPECT_EQ( + 0, WebRtcOpus_SetBitRate(opus_encoder_, channels_ == 1 ? 32000 : 64000)); + + // Set input audio as silence. + std::vector silence(input_samples * channels_, 0); + + // Setting DTX. + EXPECT_EQ(0, dtx ? WebRtcOpus_EnableDtx(opus_encoder_) + : WebRtcOpus_DisableDtx(opus_encoder_)); + + int16_t audio_type; + int16_t* output_data_decode = new int16_t[output_samples * channels_]; + + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(output_samples, + static_cast(EncodeDecode( + opus_encoder_, speech_data_.GetNextBlock(), opus_decoder_, + output_data_decode, &audio_type))); + // If not DTX, it should never enter DTX mode. If DTX, we do not care since + // whether it enters DTX depends on the signal type. + if (!dtx) { + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } + } + + // We input some silent segments. In DTX mode, the encoder will stop sending. + // However, DTX may happen after a while. + for (int i = 0; i < 30; ++i) { + EXPECT_EQ(output_samples, static_cast(EncodeDecode( + opus_encoder_, silence, opus_decoder_, + output_data_decode, &audio_type))); + if (!dtx) { + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } else if (encoded_bytes_ == 1) { + EXPECT_EQ(1, opus_encoder_->in_dtx_mode); + EXPECT_EQ(1, opus_decoder_->in_dtx_mode); + EXPECT_EQ(2, audio_type); // Comfort noise. + break; + } + } + + // When Opus is in DTX, it wakes up in a regular basis. It sends two packets, + // one with an arbitrary size and the other of 1-byte, then stops sending for + // a certain number of frames. + + // `max_dtx_frames` is the maximum number of frames Opus can stay in DTX. + // TODO(kwiberg): Why does this number depend on the encoding sample rate? + const int max_dtx_frames = + (encoder_sample_rate_hz_ == 16000 ? 800 : 400) / block_length_ms + 1; + + // We run `kRunTimeMs` milliseconds of pure silence. + const int kRunTimeMs = 4500; + + // We check that, after a `kCheckTimeMs` milliseconds (given that the CNG in + // Opus needs time to adapt), the absolute values of DTX decoded signal are + // bounded by `kOutputValueBound`. + const int kCheckTimeMs = 4000; + +#if defined(OPUS_FIXED_POINT) + // Fixed-point Opus generates a random (comfort) noise, which has a less + // predictable value bound than its floating-point Opus. This value depends on + // input signal, and the time window for checking the output values (between + // `kCheckTimeMs` and `kRunTimeMs`). + const uint16_t kOutputValueBound = 30; + +#else + const uint16_t kOutputValueBound = 2; +#endif + + int time = 0; + while (time < kRunTimeMs) { + // DTX mode is maintained for maximum `max_dtx_frames` frames. + int i = 0; + for (; i < max_dtx_frames; ++i) { + time += block_length_ms; + EXPECT_EQ(output_samples, static_cast(EncodeDecode( + opus_encoder_, silence, opus_decoder_, + output_data_decode, &audio_type))); + if (dtx) { + if (encoded_bytes_ > 1) + break; + EXPECT_EQ(0U, encoded_bytes_) // Send 0 byte. + << "Opus should have entered DTX mode."; + EXPECT_EQ(1, opus_encoder_->in_dtx_mode); + EXPECT_EQ(1, opus_decoder_->in_dtx_mode); + EXPECT_EQ(2, audio_type); // Comfort noise. + if (time >= kCheckTimeMs) { + CheckAudioBounded(output_data_decode, output_samples, channels_, + kOutputValueBound); + } + } else { + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } + } + + if (dtx) { + // With DTX, Opus must stop transmission for some time. + EXPECT_GT(i, 1); + } + + // We expect a normal payload. + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + + // Enters DTX again immediately. + time += block_length_ms; + EXPECT_EQ(output_samples, static_cast(EncodeDecode( + opus_encoder_, silence, opus_decoder_, + output_data_decode, &audio_type))); + if (dtx) { + EXPECT_EQ(1U, encoded_bytes_); // Send 1 byte. + EXPECT_EQ(1, opus_encoder_->in_dtx_mode); + EXPECT_EQ(1, opus_decoder_->in_dtx_mode); + EXPECT_EQ(2, audio_type); // Comfort noise. + if (time >= kCheckTimeMs) { + CheckAudioBounded(output_data_decode, output_samples, channels_, + kOutputValueBound); + } + } else { + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } + } + + silence[0] = 10000; + if (dtx) { + // Verify that encoder/decoder can jump out from DTX mode. + EXPECT_EQ(output_samples, static_cast(EncodeDecode( + opus_encoder_, silence, opus_decoder_, + output_data_decode, &audio_type))); + EXPECT_GT(encoded_bytes_, 1U); + EXPECT_EQ(0, opus_encoder_->in_dtx_mode); + EXPECT_EQ(0, opus_decoder_->in_dtx_mode); + EXPECT_EQ(0, audio_type); // Speech. + } + + // Free memory. + delete[] output_data_decode; + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +// Test if CBR does what we expect. +void OpusTest::TestCbrEffect(bool cbr, int block_length_ms) { + PrepareSpeechData(block_length_ms, 2000); + const size_t output_samples = + rtc::CheckedDivExact(decoder_sample_rate_hz_, 1000) * block_length_ms; + + int32_t max_pkt_size_diff = 0; + int32_t prev_pkt_size = 0; + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + + // Set bitrate. + EXPECT_EQ( + 0, WebRtcOpus_SetBitRate(opus_encoder_, channels_ == 1 ? 32000 : 64000)); + + // Setting CBR. + EXPECT_EQ(0, cbr ? WebRtcOpus_EnableCbr(opus_encoder_) + : WebRtcOpus_DisableCbr(opus_encoder_)); + + int16_t audio_type; + std::vector audio_out(output_samples * channels_); + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(output_samples, + static_cast( + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), + opus_decoder_, audio_out.data(), &audio_type))); + + if (prev_pkt_size > 0) { + int32_t diff = std::abs((int32_t)encoded_bytes_ - prev_pkt_size); + max_pkt_size_diff = std::max(max_pkt_size_diff, diff); + } + prev_pkt_size = rtc::checked_cast(encoded_bytes_); + } + + if (cbr) { + EXPECT_EQ(max_pkt_size_diff, 0); + } else { + EXPECT_GT(max_pkt_size_diff, 0); + } + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +// Test failing Create. +TEST(OpusTest, OpusCreateFail) { + WebRtcOpusEncInst* opus_encoder; + WebRtcOpusDecInst* opus_decoder; + + // Test to see that an invalid pointer is caught. + EXPECT_EQ(-1, WebRtcOpus_EncoderCreate(NULL, 1, 0, 48000)); + // Invalid channel number. + EXPECT_EQ(-1, WebRtcOpus_EncoderCreate(&opus_encoder, 257, 0, 48000)); + // Invalid applciation mode. + EXPECT_EQ(-1, WebRtcOpus_EncoderCreate(&opus_encoder, 1, 2, 48000)); + // Invalid sample rate. + EXPECT_EQ(-1, WebRtcOpus_EncoderCreate(&opus_encoder, 1, 0, 12345)); + + EXPECT_EQ(-1, WebRtcOpus_DecoderCreate(NULL, 1, 48000)); + // Invalid channel number. + EXPECT_EQ(-1, WebRtcOpus_DecoderCreate(&opus_decoder, 257, 48000)); + // Invalid sample rate. + EXPECT_EQ(-1, WebRtcOpus_DecoderCreate(&opus_decoder, 1, 12345)); +} + +// Test failing Free. +TEST(OpusTest, OpusFreeFail) { + // Test to see that an invalid pointer is caught. + EXPECT_EQ(-1, WebRtcOpus_EncoderFree(NULL)); + EXPECT_EQ(-1, WebRtcOpus_DecoderFree(NULL)); +} + +// Test normal Create and Free. +TEST_P(OpusTest, OpusCreateFree) { + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + EXPECT_TRUE(opus_encoder_ != NULL); + EXPECT_TRUE(opus_decoder_ != NULL); + // Free encoder and decoder memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +#define ENCODER_CTL(inst, vargs) \ + inst->encoder \ + ? opus_encoder_ctl(inst->encoder, vargs) \ + : opus_multistream_encoder_ctl(inst->multistream_encoder, vargs) + +TEST_P(OpusTest, OpusEncodeDecode) { + PrepareSpeechData(20, 20); + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + + // Set bitrate. + EXPECT_EQ( + 0, WebRtcOpus_SetBitRate(opus_encoder_, channels_ == 1 ? 32000 : 64000)); + + // Check number of channels for decoder. + EXPECT_EQ(channels_, WebRtcOpus_DecoderChannels(opus_decoder_)); + + // Check application mode. + opus_int32 app; + ENCODER_CTL(opus_encoder_, OPUS_GET_APPLICATION(&app)); + EXPECT_EQ(application_ == 0 ? OPUS_APPLICATION_VOIP : OPUS_APPLICATION_AUDIO, + app); + + // Encode & decode. + int16_t audio_type; + const int decode_samples_per_channel = + SamplesPerChannel(decoder_sample_rate_hz_, /*ms=*/20); + int16_t* output_data_decode = + new int16_t[decode_samples_per_channel * channels_]; + EXPECT_EQ(decode_samples_per_channel, + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), + opus_decoder_, output_data_decode, &audio_type)); + + // Free memory. + delete[] output_data_decode; + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusSetBitRate) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetBitRate(opus_encoder_, 60000)); + + // Create encoder memory, try with different bitrates. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, 30000)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, 60000)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, 300000)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, 600000)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusSetComplexity) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetComplexity(opus_encoder_, 9)); + + // Create encoder memory, try with different complexities. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_encoder_, 0)); + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_encoder_, 10)); + EXPECT_EQ(-1, WebRtcOpus_SetComplexity(opus_encoder_, 11)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusSetBandwidth) { + if (channels_ > 2) { + // TODO(webrtc:10217): investigate why multi-stream Opus reports + // narrowband when it's configured with FULLBAND. + return; + } + PrepareSpeechData(20, 20); + + int16_t audio_type; + const int decode_samples_per_channel = + SamplesPerChannel(decoder_sample_rate_hz_, /*ms=*/20); + std::unique_ptr output_data_decode( + new int16_t[decode_samples_per_channel * channels_]()); + + // Test without creating encoder memory. + EXPECT_EQ(-1, + WebRtcOpus_SetBandwidth(opus_encoder_, OPUS_BANDWIDTH_NARROWBAND)); + EXPECT_EQ(-1, WebRtcOpus_GetBandwidth(opus_encoder_)); + + // Create encoder memory, try with different bandwidths. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + + EXPECT_EQ(-1, WebRtcOpus_SetBandwidth(opus_encoder_, + OPUS_BANDWIDTH_NARROWBAND - 1)); + EXPECT_EQ(0, + WebRtcOpus_SetBandwidth(opus_encoder_, OPUS_BANDWIDTH_NARROWBAND)); + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), opus_decoder_, + output_data_decode.get(), &audio_type); + EXPECT_EQ(OPUS_BANDWIDTH_NARROWBAND, WebRtcOpus_GetBandwidth(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_SetBandwidth(opus_encoder_, OPUS_BANDWIDTH_FULLBAND)); + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), opus_decoder_, + output_data_decode.get(), &audio_type); + EXPECT_EQ(encoder_sample_rate_hz_ == 16000 ? OPUS_BANDWIDTH_WIDEBAND + : OPUS_BANDWIDTH_FULLBAND, + WebRtcOpus_GetBandwidth(opus_encoder_)); + EXPECT_EQ( + -1, WebRtcOpus_SetBandwidth(opus_encoder_, OPUS_BANDWIDTH_FULLBAND + 1)); + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), opus_decoder_, + output_data_decode.get(), &audio_type); + EXPECT_EQ(encoder_sample_rate_hz_ == 16000 ? OPUS_BANDWIDTH_WIDEBAND + : OPUS_BANDWIDTH_FULLBAND, + WebRtcOpus_GetBandwidth(opus_encoder_)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusForceChannels) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetForceChannels(opus_encoder_, 1)); + + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + ASSERT_NE(nullptr, opus_encoder_); + + if (channels_ >= 2) { + EXPECT_EQ(-1, WebRtcOpus_SetForceChannels(opus_encoder_, 3)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 2)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 1)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 0)); + } else { + EXPECT_EQ(-1, WebRtcOpus_SetForceChannels(opus_encoder_, 2)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 1)); + EXPECT_EQ(0, WebRtcOpus_SetForceChannels(opus_encoder_, 0)); + } + + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +// Encode and decode one frame, initialize the decoder and +// decode once more. +TEST_P(OpusTest, OpusDecodeInit) { + PrepareSpeechData(20, 20); + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + + // Encode & decode. + int16_t audio_type; + const int decode_samples_per_channel = + SamplesPerChannel(decoder_sample_rate_hz_, /*ms=*/20); + int16_t* output_data_decode = + new int16_t[decode_samples_per_channel * channels_]; + EXPECT_EQ(decode_samples_per_channel, + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), + opus_decoder_, output_data_decode, &audio_type)); + + WebRtcOpus_DecoderInit(opus_decoder_); + + EXPECT_EQ(decode_samples_per_channel, + WebRtcOpus_Decode(opus_decoder_, bitstream_, encoded_bytes_, + output_data_decode, &audio_type)); + + // Free memory. + delete[] output_data_decode; + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusEnableDisableFec) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_EnableFec(opus_encoder_)); + EXPECT_EQ(-1, WebRtcOpus_DisableFec(opus_encoder_)); + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DisableFec(opus_encoder_)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusEnableDisableDtx) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_EnableDtx(opus_encoder_)); + EXPECT_EQ(-1, WebRtcOpus_DisableDtx(opus_encoder_)); + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + + opus_int32 dtx; + + // DTX is off by default. + ENCODER_CTL(opus_encoder_, OPUS_GET_DTX(&dtx)); + EXPECT_EQ(0, dtx); + + // Test to enable DTX. + EXPECT_EQ(0, WebRtcOpus_EnableDtx(opus_encoder_)); + ENCODER_CTL(opus_encoder_, OPUS_GET_DTX(&dtx)); + EXPECT_EQ(1, dtx); + + // Test to disable DTX. + EXPECT_EQ(0, WebRtcOpus_DisableDtx(opus_encoder_)); + ENCODER_CTL(opus_encoder_, OPUS_GET_DTX(&dtx)); + EXPECT_EQ(0, dtx); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusDtxOff) { + TestDtxEffect(false, 10); + TestDtxEffect(false, 20); + TestDtxEffect(false, 40); +} + +TEST_P(OpusTest, OpusDtxOn) { + if (channels_ > 2 || application_ != 0) { + // DTX does not work with OPUS_APPLICATION_AUDIO at low complexity settings. + // TODO(webrtc:10218): adapt the test to the sizes and order of multi-stream + // DTX packets. + return; + } + TestDtxEffect(true, 10); + TestDtxEffect(true, 20); + TestDtxEffect(true, 40); +} + +TEST_P(OpusTest, OpusCbrOff) { + TestCbrEffect(false, 10); + TestCbrEffect(false, 20); + TestCbrEffect(false, 40); +} + +TEST_P(OpusTest, OpusCbrOn) { + TestCbrEffect(true, 10); + TestCbrEffect(true, 20); + TestCbrEffect(true, 40); +} + +TEST_P(OpusTest, OpusSetPacketLossRate) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_encoder_, 50)); + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate(opus_encoder_, 50)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_encoder_, -1)); + EXPECT_EQ(-1, WebRtcOpus_SetPacketLossRate(opus_encoder_, 101)); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +TEST_P(OpusTest, OpusSetMaxPlaybackRate) { + // Test without creating encoder memory. + EXPECT_EQ(-1, WebRtcOpus_SetMaxPlaybackRate(opus_encoder_, 20000)); + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_FULLBAND, 48000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_FULLBAND, 24001); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_SUPERWIDEBAND, 24000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_SUPERWIDEBAND, 16001); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_WIDEBAND, 16000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_WIDEBAND, 12001); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_MEDIUMBAND, 12000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_MEDIUMBAND, 8001); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_NARROWBAND, 8000); + SetMaxPlaybackRate(opus_encoder_, OPUS_BANDWIDTH_NARROWBAND, 4000); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); +} + +// Test PLC. +TEST_P(OpusTest, OpusDecodePlc) { + PrepareSpeechData(20, 20); + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + + // Set bitrate. + EXPECT_EQ( + 0, WebRtcOpus_SetBitRate(opus_encoder_, channels_ == 1 ? 32000 : 64000)); + + // Check number of channels for decoder. + EXPECT_EQ(channels_, WebRtcOpus_DecoderChannels(opus_decoder_)); + + // Encode & decode. + int16_t audio_type; + const int decode_samples_per_channel = + SamplesPerChannel(decoder_sample_rate_hz_, /*ms=*/20); + int16_t* output_data_decode = + new int16_t[decode_samples_per_channel * channels_]; + EXPECT_EQ(decode_samples_per_channel, + EncodeDecode(opus_encoder_, speech_data_.GetNextBlock(), + opus_decoder_, output_data_decode, &audio_type)); + + // Call decoder PLC. + constexpr int kPlcDurationMs = 10; + const int plc_samples = decoder_sample_rate_hz_ * kPlcDurationMs / 1000; + int16_t* plc_buffer = new int16_t[plc_samples * channels_]; + EXPECT_EQ(plc_samples, + WebRtcOpus_Decode(opus_decoder_, NULL, 0, plc_buffer, &audio_type)); + + // Free memory. + delete[] plc_buffer; + delete[] output_data_decode; + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +// Duration estimation. +TEST_P(OpusTest, OpusDurationEstimation) { + PrepareSpeechData(20, 20); + + // Create. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + + // 10 ms. We use only first 10 ms of a 20 ms block. + auto speech_block = speech_data_.GetNextBlock(); + int encoded_bytes_int = WebRtcOpus_Encode( + opus_encoder_, speech_block.data(), + rtc::CheckedDivExact(speech_block.size(), 2 * channels_), kMaxBytes, + bitstream_); + EXPECT_GE(encoded_bytes_int, 0); + EXPECT_EQ(SamplesPerChannel(decoder_sample_rate_hz_, /*ms=*/10), + WebRtcOpus_DurationEst(opus_decoder_, bitstream_, + static_cast(encoded_bytes_int))); + + // 20 ms + speech_block = speech_data_.GetNextBlock(); + encoded_bytes_int = + WebRtcOpus_Encode(opus_encoder_, speech_block.data(), + rtc::CheckedDivExact(speech_block.size(), channels_), + kMaxBytes, bitstream_); + EXPECT_GE(encoded_bytes_int, 0); + EXPECT_EQ(SamplesPerChannel(decoder_sample_rate_hz_, /*ms=*/20), + WebRtcOpus_DurationEst(opus_decoder_, bitstream_, + static_cast(encoded_bytes_int))); + + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST_P(OpusTest, OpusDecodeRepacketized) { + if (channels_ > 2) { + // As per the Opus documentation + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__repacketizer.html#details, + // multiple streams are not supported. + return; + } + constexpr size_t kPackets = 6; + + PrepareSpeechData(20, 20 * kPackets); + + // Create encoder memory. + CreateSingleOrMultiStreamEncoder(&opus_encoder_, channels_, application_, + use_multistream_, encoder_sample_rate_hz_); + ASSERT_NE(nullptr, opus_encoder_); + CreateSingleOrMultiStreamDecoder(&opus_decoder_, channels_, use_multistream_, + decoder_sample_rate_hz_); + ASSERT_NE(nullptr, opus_decoder_); + + // Set bitrate. + EXPECT_EQ( + 0, WebRtcOpus_SetBitRate(opus_encoder_, channels_ == 1 ? 32000 : 64000)); + + // Check number of channels for decoder. + EXPECT_EQ(channels_, WebRtcOpus_DecoderChannels(opus_decoder_)); + + // Encode & decode. + int16_t audio_type; + const int decode_samples_per_channel = + SamplesPerChannel(decoder_sample_rate_hz_, /*ms=*/20); + std::unique_ptr output_data_decode( + new int16_t[kPackets * decode_samples_per_channel * channels_]); + OpusRepacketizer* rp = opus_repacketizer_create(); + + size_t num_packets = 0; + constexpr size_t kMaxCycles = 100; + for (size_t idx = 0; idx < kMaxCycles; ++idx) { + auto speech_block = speech_data_.GetNextBlock(); + encoded_bytes_ = + WebRtcOpus_Encode(opus_encoder_, speech_block.data(), + rtc::CheckedDivExact(speech_block.size(), channels_), + kMaxBytes, bitstream_); + if (opus_repacketizer_cat(rp, bitstream_, + rtc::checked_cast(encoded_bytes_)) == + OPUS_OK) { + ++num_packets; + if (num_packets == kPackets) { + break; + } + } else { + // Opus repacketizer cannot guarantee a success. We try again if it fails. + opus_repacketizer_init(rp); + num_packets = 0; + } + } + EXPECT_EQ(kPackets, num_packets); + + encoded_bytes_ = opus_repacketizer_out(rp, bitstream_, kMaxBytes); + + EXPECT_EQ(decode_samples_per_channel * kPackets, + static_cast(WebRtcOpus_DurationEst( + opus_decoder_, bitstream_, encoded_bytes_))); + + EXPECT_EQ(decode_samples_per_channel * kPackets, + static_cast( + WebRtcOpus_Decode(opus_decoder_, bitstream_, encoded_bytes_, + output_data_decode.get(), &audio_type))); + + // Free memory. + opus_repacketizer_destroy(rp); + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); +} + +TEST(OpusVadTest, CeltUnknownStatus) { + const uint8_t celt[] = {0x80}; + EXPECT_EQ(WebRtcOpus_PacketHasVoiceActivity(celt, 1), -1); +} + +TEST(OpusVadTest, Mono20msVadSet) { + uint8_t silk20msMonoVad[] = {0x78, 0x80}; + EXPECT_TRUE(WebRtcOpus_PacketHasVoiceActivity(silk20msMonoVad, 2)); +} + +TEST(OpusVadTest, Mono20MsVadUnset) { + uint8_t silk20msMonoSilence[] = {0x78, 0x00}; + EXPECT_FALSE(WebRtcOpus_PacketHasVoiceActivity(silk20msMonoSilence, 2)); +} + +TEST(OpusVadTest, Stereo20MsVadOnSideChannel) { + uint8_t silk20msStereoVadSideChannel[] = {0x78 | 0x04, 0x20}; + EXPECT_TRUE( + WebRtcOpus_PacketHasVoiceActivity(silk20msStereoVadSideChannel, 2)); +} + +TEST(OpusVadTest, TwoOpusMonoFramesVadOnSecond) { + uint8_t twoMonoFrames[] = {0x78 | 0x1, 0x00, 0x80}; + EXPECT_TRUE(WebRtcOpus_PacketHasVoiceActivity(twoMonoFrames, 3)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/BUILD.gn b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/BUILD.gn new file mode 100644 index 0000000000..8bc0bf5e0e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/BUILD.gn @@ -0,0 +1,55 @@ +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../../../webrtc.gni") + +visibility = [ + ":*", + "../../../:*", +] + +if (rtc_include_tests) { + rtc_library("test") { + testonly = true + + sources = [ + "audio_ring_buffer.cc", + "audio_ring_buffer.h", + "blocker.cc", + "blocker.h", + "lapped_transform.cc", + "lapped_transform.h", + ] + + deps = [ + "../../../../../common_audio", + "../../../../../common_audio:common_audio_c", + "../../../../../rtc_base:checks", + "../../../../../rtc_base/memory:aligned_malloc", + ] + } + + rtc_library("test_unittest") { + testonly = true + + sources = [ + "audio_ring_buffer_unittest.cc", + "blocker_unittest.cc", + "lapped_transform_unittest.cc", + ] + + deps = [ + ":test", + "../../../../../common_audio", + "../../../../../common_audio:common_audio_c", + "../../../../../rtc_base:macromagic", + "../../../../../test:test_support", + "//testing/gtest", + ] + } +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer.cc new file mode 100644 index 0000000000..2a71b43d2c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer.cc @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/test/audio_ring_buffer.h" + +#include "common_audio/ring_buffer.h" +#include "rtc_base/checks.h" + +// This is a simple multi-channel wrapper over the ring_buffer.h C interface. + +namespace webrtc { + +AudioRingBuffer::AudioRingBuffer(size_t channels, size_t max_frames) { + buffers_.reserve(channels); + for (size_t i = 0; i < channels; ++i) + buffers_.push_back(WebRtc_CreateBuffer(max_frames, sizeof(float))); +} + +AudioRingBuffer::~AudioRingBuffer() { + for (auto* buf : buffers_) + WebRtc_FreeBuffer(buf); +} + +void AudioRingBuffer::Write(const float* const* data, + size_t channels, + size_t frames) { + RTC_DCHECK_EQ(buffers_.size(), channels); + for (size_t i = 0; i < channels; ++i) { + const size_t written = WebRtc_WriteBuffer(buffers_[i], data[i], frames); + RTC_CHECK_EQ(written, frames); + } +} + +void AudioRingBuffer::Read(float* const* data, size_t channels, size_t frames) { + RTC_DCHECK_EQ(buffers_.size(), channels); + for (size_t i = 0; i < channels; ++i) { + const size_t read = + WebRtc_ReadBuffer(buffers_[i], nullptr, data[i], frames); + RTC_CHECK_EQ(read, frames); + } +} + +size_t AudioRingBuffer::ReadFramesAvailable() const { + // All buffers have the same amount available. + return WebRtc_available_read(buffers_[0]); +} + +size_t AudioRingBuffer::WriteFramesAvailable() const { + // All buffers have the same amount available. + return WebRtc_available_write(buffers_[0]); +} + +void AudioRingBuffer::MoveReadPositionForward(size_t frames) { + for (auto* buf : buffers_) { + const size_t moved = + static_cast(WebRtc_MoveReadPtr(buf, static_cast(frames))); + RTC_CHECK_EQ(moved, frames); + } +} + +void AudioRingBuffer::MoveReadPositionBackward(size_t frames) { + for (auto* buf : buffers_) { + const size_t moved = static_cast( + -WebRtc_MoveReadPtr(buf, -static_cast(frames))); + RTC_CHECK_EQ(moved, frames); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer.h new file mode 100644 index 0000000000..a280ca2410 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_TEST_AUDIO_RING_BUFFER_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_TEST_AUDIO_RING_BUFFER_H_ + +#include + +#include +#include + +struct RingBuffer; + +namespace webrtc { + +// A ring buffer tailored for float deinterleaved audio. Any operation that +// cannot be performed as requested will cause a crash (e.g. insufficient data +// in the buffer to fulfill a read request.) +class AudioRingBuffer final { + public: + // Specify the number of channels and maximum number of frames the buffer will + // contain. + AudioRingBuffer(size_t channels, size_t max_frames); + ~AudioRingBuffer(); + + // Copies `data` to the buffer and advances the write pointer. `channels` must + // be the same as at creation time. + void Write(const float* const* data, size_t channels, size_t frames); + + // Copies from the buffer to `data` and advances the read pointer. `channels` + // must be the same as at creation time. + void Read(float* const* data, size_t channels, size_t frames); + + size_t ReadFramesAvailable() const; + size_t WriteFramesAvailable() const; + + // Moves the read position. The forward version advances the read pointer + // towards the write pointer and the backward verison withdraws the read + // pointer away from the write pointer (i.e. flushing and stuffing the buffer + // respectively.) + void MoveReadPositionForward(size_t frames); + void MoveReadPositionBackward(size_t frames); + + private: + // TODO(kwiberg): Use std::vector> instead. + std::vector buffers_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_TEST_AUDIO_RING_BUFFER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer_unittest.cc new file mode 100644 index 0000000000..6dbc8ee9fe --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/audio_ring_buffer_unittest.cc @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/test/audio_ring_buffer.h" + +#include + +#include "common_audio/channel_buffer.h" +#include "test/gtest.h" + +namespace webrtc { + +class AudioRingBufferTest + : public ::testing::TestWithParam< ::testing::tuple > { +}; + +void ReadAndWriteTest(const ChannelBuffer& input, + size_t num_write_chunk_frames, + size_t num_read_chunk_frames, + size_t buffer_frames, + ChannelBuffer* output) { + const size_t num_channels = input.num_channels(); + const size_t total_frames = input.num_frames(); + AudioRingBuffer buf(num_channels, buffer_frames); + std::unique_ptr slice(new float*[num_channels]); + + size_t input_pos = 0; + size_t output_pos = 0; + while (input_pos + buf.WriteFramesAvailable() < total_frames) { + // Write until the buffer is as full as possible. + while (buf.WriteFramesAvailable() >= num_write_chunk_frames) { + buf.Write(input.Slice(slice.get(), input_pos), num_channels, + num_write_chunk_frames); + input_pos += num_write_chunk_frames; + } + // Read until the buffer is as empty as possible. + while (buf.ReadFramesAvailable() >= num_read_chunk_frames) { + EXPECT_LT(output_pos, total_frames); + buf.Read(output->Slice(slice.get(), output_pos), num_channels, + num_read_chunk_frames); + output_pos += num_read_chunk_frames; + } + } + + // Write and read the last bit. + if (input_pos < total_frames) { + buf.Write(input.Slice(slice.get(), input_pos), num_channels, + total_frames - input_pos); + } + if (buf.ReadFramesAvailable()) { + buf.Read(output->Slice(slice.get(), output_pos), num_channels, + buf.ReadFramesAvailable()); + } + EXPECT_EQ(0u, buf.ReadFramesAvailable()); +} + +TEST_P(AudioRingBufferTest, ReadDataMatchesWrittenData) { + const size_t kFrames = 5000; + const size_t num_channels = ::testing::get<3>(GetParam()); + + // Initialize the input data to an increasing sequence. + ChannelBuffer input(kFrames, static_cast(num_channels)); + for (size_t i = 0; i < num_channels; ++i) + for (size_t j = 0; j < kFrames; ++j) + input.channels()[i][j] = (i + 1) * (j + 1); + + ChannelBuffer output(kFrames, static_cast(num_channels)); + ReadAndWriteTest(input, ::testing::get<0>(GetParam()), + ::testing::get<1>(GetParam()), ::testing::get<2>(GetParam()), + &output); + + // Verify the read data matches the input. + for (size_t i = 0; i < num_channels; ++i) + for (size_t j = 0; j < kFrames; ++j) + EXPECT_EQ(input.channels()[i][j], output.channels()[i][j]); +} + +INSTANTIATE_TEST_SUITE_P( + AudioRingBufferTest, + AudioRingBufferTest, + ::testing::Combine(::testing::Values(10, 20, 42), // num_write_chunk_frames + ::testing::Values(1, 10, 17), // num_read_chunk_frames + ::testing::Values(100, 256), // buffer_frames + ::testing::Values(1, 4))); // num_channels + +TEST_F(AudioRingBufferTest, MoveReadPosition) { + const size_t kNumChannels = 1; + const float kInputArray[] = {1, 2, 3, 4}; + const size_t kNumFrames = sizeof(kInputArray) / sizeof(*kInputArray); + ChannelBuffer input(kNumFrames, kNumChannels); + input.SetDataForTesting(kInputArray, kNumFrames); + AudioRingBuffer buf(kNumChannels, kNumFrames); + buf.Write(input.channels(), kNumChannels, kNumFrames); + + buf.MoveReadPositionForward(3); + ChannelBuffer output(1, kNumChannels); + buf.Read(output.channels(), kNumChannels, 1); + EXPECT_EQ(4, output.channels()[0][0]); + buf.MoveReadPositionBackward(3); + buf.Read(output.channels(), kNumChannels, 1); + EXPECT_EQ(2, output.channels()[0][0]); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker.cc new file mode 100644 index 0000000000..33406cead9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker.cc @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/test/blocker.h" + +#include + +#include "rtc_base/checks.h" + +namespace { + +// Adds `a` and `b` frame by frame into `result` (basically matrix addition). +void AddFrames(const float* const* a, + size_t a_start_index, + const float* const* b, + int b_start_index, + size_t num_frames, + size_t num_channels, + float* const* result, + size_t result_start_index) { + for (size_t i = 0; i < num_channels; ++i) { + for (size_t j = 0; j < num_frames; ++j) { + result[i][j + result_start_index] = + a[i][j + a_start_index] + b[i][j + b_start_index]; + } + } +} + +// Copies `src` into `dst` channel by channel. +void CopyFrames(const float* const* src, + size_t src_start_index, + size_t num_frames, + size_t num_channels, + float* const* dst, + size_t dst_start_index) { + for (size_t i = 0; i < num_channels; ++i) { + memcpy(&dst[i][dst_start_index], &src[i][src_start_index], + num_frames * sizeof(dst[i][dst_start_index])); + } +} + +// Moves `src` into `dst` channel by channel. +void MoveFrames(const float* const* src, + size_t src_start_index, + size_t num_frames, + size_t num_channels, + float* const* dst, + size_t dst_start_index) { + for (size_t i = 0; i < num_channels; ++i) { + memmove(&dst[i][dst_start_index], &src[i][src_start_index], + num_frames * sizeof(dst[i][dst_start_index])); + } +} + +void ZeroOut(float* const* buffer, + size_t starting_idx, + size_t num_frames, + size_t num_channels) { + for (size_t i = 0; i < num_channels; ++i) { + memset(&buffer[i][starting_idx], 0, + num_frames * sizeof(buffer[i][starting_idx])); + } +} + +// Pointwise multiplies each channel of `frames` with `window`. Results are +// stored in `frames`. +void ApplyWindow(const float* window, + size_t num_frames, + size_t num_channels, + float* const* frames) { + for (size_t i = 0; i < num_channels; ++i) { + for (size_t j = 0; j < num_frames; ++j) { + frames[i][j] = frames[i][j] * window[j]; + } + } +} + +size_t gcd(size_t a, size_t b) { + size_t tmp; + while (b) { + tmp = a; + a = b; + b = tmp % b; + } + return a; +} + +} // namespace + +namespace webrtc { + +Blocker::Blocker(size_t chunk_size, + size_t block_size, + size_t num_input_channels, + size_t num_output_channels, + const float* window, + size_t shift_amount, + BlockerCallback* callback) + : chunk_size_(chunk_size), + block_size_(block_size), + num_input_channels_(num_input_channels), + num_output_channels_(num_output_channels), + initial_delay_(block_size_ - gcd(chunk_size, shift_amount)), + frame_offset_(0), + input_buffer_(num_input_channels_, chunk_size_ + initial_delay_), + output_buffer_(chunk_size_ + initial_delay_, num_output_channels_), + input_block_(block_size_, num_input_channels_), + output_block_(block_size_, num_output_channels_), + window_(new float[block_size_]), + shift_amount_(shift_amount), + callback_(callback) { + RTC_CHECK_LE(num_output_channels_, num_input_channels_); + RTC_CHECK_LE(shift_amount_, block_size_); + + memcpy(window_.get(), window, block_size_ * sizeof(*window_.get())); + input_buffer_.MoveReadPositionBackward(initial_delay_); +} + +Blocker::~Blocker() = default; + +// When block_size < chunk_size the input and output buffers look like this: +// +// delay* chunk_size chunk_size + delay* +// buffer: <-------------|---------------------|---------------|> +// _a_ _b_ _c_ +// +// On each call to ProcessChunk(): +// 1. New input gets read into sections _b_ and _c_ of the input buffer. +// 2. We block starting from frame_offset. +// 3. We block until we reach a block `bl` that doesn't contain any frames +// from sections _a_ or _b_ of the input buffer. +// 4. We window the current block, fire the callback for processing, window +// again, and overlap/add to the output buffer. +// 5. We copy sections _a_ and _b_ of the output buffer into output. +// 6. For both the input and the output buffers, we copy section _c_ into +// section _a_. +// 7. We set the new frame_offset to be the difference between the first frame +// of `bl` and the border between sections _b_ and _c_. +// +// When block_size > chunk_size the input and output buffers look like this: +// +// chunk_size delay* chunk_size + delay* +// buffer: <-------------|---------------------|---------------|> +// _a_ _b_ _c_ +// +// On each call to ProcessChunk(): +// The procedure is the same as above, except for: +// 1. New input gets read into section _c_ of the input buffer. +// 3. We block until we reach a block `bl` that doesn't contain any frames +// from section _a_ of the input buffer. +// 5. We copy section _a_ of the output buffer into output. +// 6. For both the input and the output buffers, we copy sections _b_ and _c_ +// into section _a_ and _b_. +// 7. We set the new frame_offset to be the difference between the first frame +// of `bl` and the border between sections _a_ and _b_. +// +// * delay here refers to inintial_delay_ +// +// TODO(claguna): Look at using ring buffers to eliminate some copies. +void Blocker::ProcessChunk(const float* const* input, + size_t chunk_size, + size_t num_input_channels, + size_t num_output_channels, + float* const* output) { + RTC_CHECK_EQ(chunk_size, chunk_size_); + RTC_CHECK_EQ(num_input_channels, num_input_channels_); + RTC_CHECK_EQ(num_output_channels, num_output_channels_); + + input_buffer_.Write(input, num_input_channels, chunk_size_); + size_t first_frame_in_block = frame_offset_; + + // Loop through blocks. + while (first_frame_in_block < chunk_size_) { + input_buffer_.Read(input_block_.channels(), num_input_channels, + block_size_); + input_buffer_.MoveReadPositionBackward(block_size_ - shift_amount_); + + ApplyWindow(window_.get(), block_size_, num_input_channels_, + input_block_.channels()); + callback_->ProcessBlock(input_block_.channels(), block_size_, + num_input_channels_, num_output_channels_, + output_block_.channels()); + ApplyWindow(window_.get(), block_size_, num_output_channels_, + output_block_.channels()); + + AddFrames(output_buffer_.channels(), first_frame_in_block, + output_block_.channels(), 0, block_size_, num_output_channels_, + output_buffer_.channels(), first_frame_in_block); + + first_frame_in_block += shift_amount_; + } + + // Copy output buffer to output + CopyFrames(output_buffer_.channels(), 0, chunk_size_, num_output_channels_, + output, 0); + + // Copy output buffer [chunk_size_, chunk_size_ + initial_delay] + // to output buffer [0, initial_delay], zero the rest. + MoveFrames(output_buffer_.channels(), chunk_size, initial_delay_, + num_output_channels_, output_buffer_.channels(), 0); + ZeroOut(output_buffer_.channels(), initial_delay_, chunk_size_, + num_output_channels_); + + // Calculate new starting frames. + frame_offset_ = first_frame_in_block - chunk_size_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker.h new file mode 100644 index 0000000000..59b7e29621 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_TEST_BLOCKER_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_TEST_BLOCKER_H_ + +#include + +#include "common_audio/channel_buffer.h" +#include "modules/audio_coding/codecs/opus/test/audio_ring_buffer.h" + +namespace webrtc { + +// The callback function to process audio in the time domain. Input has already +// been windowed, and output will be windowed. The number of input channels +// must be >= the number of output channels. +class BlockerCallback { + public: + virtual ~BlockerCallback() {} + + virtual void ProcessBlock(const float* const* input, + size_t num_frames, + size_t num_input_channels, + size_t num_output_channels, + float* const* output) = 0; +}; + +// The main purpose of Blocker is to abstract away the fact that often we +// receive a different number of audio frames than our transform takes. For +// example, most FFTs work best when the fft-size is a power of 2, but suppose +// we receive 20ms of audio at a sample rate of 48000. That comes to 960 frames +// of audio, which is not a power of 2. Blocker allows us to specify the +// transform and all other necessary processing via the Process() callback +// function without any constraints on the transform-size +// (read: `block_size_`) or received-audio-size (read: `chunk_size_`). +// We handle this for the multichannel audio case, allowing for different +// numbers of input and output channels (for example, beamforming takes 2 or +// more input channels and returns 1 output channel). Audio signals are +// represented as deinterleaved floats in the range [-1, 1]. +// +// Blocker is responsible for: +// - blocking audio while handling potential discontinuities on the edges +// of chunks +// - windowing blocks before sending them to Process() +// - windowing processed blocks, and overlap-adding them together before +// sending back a processed chunk +// +// To use blocker: +// 1. Impelment a BlockerCallback object `bc`. +// 2. Instantiate a Blocker object `b`, passing in `bc`. +// 3. As you receive audio, call b.ProcessChunk() to get processed audio. +// +// A small amount of delay is added to the first received chunk to deal with +// the difference in chunk/block sizes. This delay is <= chunk_size. +// +// Ownership of window is retained by the caller. That is, Blocker makes a +// copy of window and does not attempt to delete it. +class Blocker { + public: + Blocker(size_t chunk_size, + size_t block_size, + size_t num_input_channels, + size_t num_output_channels, + const float* window, + size_t shift_amount, + BlockerCallback* callback); + ~Blocker(); + + void ProcessChunk(const float* const* input, + size_t chunk_size, + size_t num_input_channels, + size_t num_output_channels, + float* const* output); + + size_t initial_delay() const { return initial_delay_; } + + private: + const size_t chunk_size_; + const size_t block_size_; + const size_t num_input_channels_; + const size_t num_output_channels_; + + // The number of frames of delay to add at the beginning of the first chunk. + const size_t initial_delay_; + + // The frame index into the input buffer where the first block should be read + // from. This is necessary because shift_amount_ is not necessarily a + // multiple of chunk_size_, so blocks won't line up at the start of the + // buffer. + size_t frame_offset_; + + // Since blocks nearly always overlap, there are certain blocks that require + // frames from the end of one chunk and the beginning of the next chunk. The + // input and output buffers are responsible for saving those frames between + // calls to ProcessChunk(). + // + // Both contain |initial delay| + `chunk_size` frames. The input is a fairly + // standard FIFO, but due to the overlap-add it's harder to use an + // AudioRingBuffer for the output. + AudioRingBuffer input_buffer_; + ChannelBuffer output_buffer_; + + // Space for the input block (can't wrap because of windowing). + ChannelBuffer input_block_; + + // Space for the output block (can't wrap because of overlap/add). + ChannelBuffer output_block_; + + std::unique_ptr window_; + + // The amount of frames between the start of contiguous blocks. For example, + // `shift_amount_` = `block_size_` / 2 for a Hann window. + size_t shift_amount_; + + BlockerCallback* callback_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_TEST_BLOCKER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker_unittest.cc new file mode 100644 index 0000000000..9c8e789ba9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/blocker_unittest.cc @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/test/blocker.h" + +#include + +#include "rtc_base/arraysize.h" +#include "test/gtest.h" + +namespace { + +// Callback Function to add 3 to every sample in the signal. +class PlusThreeBlockerCallback : public webrtc::BlockerCallback { + public: + void ProcessBlock(const float* const* input, + size_t num_frames, + size_t num_input_channels, + size_t num_output_channels, + float* const* output) override { + for (size_t i = 0; i < num_output_channels; ++i) { + for (size_t j = 0; j < num_frames; ++j) { + output[i][j] = input[i][j] + 3; + } + } + } +}; + +// No-op Callback Function. +class CopyBlockerCallback : public webrtc::BlockerCallback { + public: + void ProcessBlock(const float* const* input, + size_t num_frames, + size_t num_input_channels, + size_t num_output_channels, + float* const* output) override { + for (size_t i = 0; i < num_output_channels; ++i) { + for (size_t j = 0; j < num_frames; ++j) { + output[i][j] = input[i][j]; + } + } + } +}; + +} // namespace + +namespace webrtc { + +// Tests blocking with a window that multiplies the signal by 2, a callback +// that adds 3 to each sample in the signal, and different combinations of chunk +// size, block size, and shift amount. +class BlockerTest : public ::testing::Test { + protected: + void RunTest(Blocker* blocker, + size_t chunk_size, + size_t num_frames, + const float* const* input, + float* const* input_chunk, + float* const* output, + float* const* output_chunk, + size_t num_input_channels, + size_t num_output_channels) { + size_t start = 0; + size_t end = chunk_size - 1; + while (end < num_frames) { + CopyTo(input_chunk, 0, start, num_input_channels, chunk_size, input); + blocker->ProcessChunk(input_chunk, chunk_size, num_input_channels, + num_output_channels, output_chunk); + CopyTo(output, start, 0, num_output_channels, chunk_size, output_chunk); + + start += chunk_size; + end += chunk_size; + } + } + + void ValidateSignalEquality(const float* const* expected, + const float* const* actual, + size_t num_channels, + size_t num_frames) { + for (size_t i = 0; i < num_channels; ++i) { + for (size_t j = 0; j < num_frames; ++j) { + EXPECT_FLOAT_EQ(expected[i][j], actual[i][j]); + } + } + } + + void ValidateInitialDelay(const float* const* output, + size_t num_channels, + size_t num_frames, + size_t initial_delay) { + for (size_t i = 0; i < num_channels; ++i) { + for (size_t j = 0; j < num_frames; ++j) { + if (j < initial_delay) { + EXPECT_FLOAT_EQ(output[i][j], 0.f); + } else { + EXPECT_GT(output[i][j], 0.f); + } + } + } + } + + static void CopyTo(float* const* dst, + size_t start_index_dst, + size_t start_index_src, + size_t num_channels, + size_t num_frames, + const float* const* src) { + for (size_t i = 0; i < num_channels; ++i) { + memcpy(&dst[i][start_index_dst], &src[i][start_index_src], + num_frames * sizeof(float)); + } + } +}; + +TEST_F(BlockerTest, TestBlockerMutuallyPrimeChunkandBlockSize) { + const size_t kNumInputChannels = 3; + const size_t kNumOutputChannels = 2; + const size_t kNumFrames = 10; + const size_t kBlockSize = 4; + const size_t kChunkSize = 5; + const size_t kShiftAmount = 2; + + const float kInput[kNumInputChannels][kNumFrames] = { + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + {3, 3, 3, 3, 3, 3, 3, 3, 3, 3}}; + ChannelBuffer input_cb(kNumFrames, kNumInputChannels); + input_cb.SetDataForTesting(kInput[0], sizeof(kInput) / sizeof(**kInput)); + + const float kExpectedOutput[kNumInputChannels][kNumFrames] = { + {6, 6, 12, 20, 20, 20, 20, 20, 20, 20}, + {6, 6, 12, 28, 28, 28, 28, 28, 28, 28}}; + ChannelBuffer expected_output_cb(kNumFrames, kNumInputChannels); + expected_output_cb.SetDataForTesting( + kExpectedOutput[0], sizeof(kExpectedOutput) / sizeof(**kExpectedOutput)); + + const float kWindow[kBlockSize] = {2.f, 2.f, 2.f, 2.f}; + + ChannelBuffer actual_output_cb(kNumFrames, kNumOutputChannels); + ChannelBuffer input_chunk_cb(kChunkSize, kNumInputChannels); + ChannelBuffer output_chunk_cb(kChunkSize, kNumOutputChannels); + + PlusThreeBlockerCallback callback; + Blocker blocker(kChunkSize, kBlockSize, kNumInputChannels, kNumOutputChannels, + kWindow, kShiftAmount, &callback); + + RunTest(&blocker, kChunkSize, kNumFrames, input_cb.channels(), + input_chunk_cb.channels(), actual_output_cb.channels(), + output_chunk_cb.channels(), kNumInputChannels, kNumOutputChannels); + + ValidateSignalEquality(expected_output_cb.channels(), + actual_output_cb.channels(), kNumOutputChannels, + kNumFrames); +} + +TEST_F(BlockerTest, TestBlockerMutuallyPrimeShiftAndBlockSize) { + const size_t kNumInputChannels = 3; + const size_t kNumOutputChannels = 2; + const size_t kNumFrames = 12; + const size_t kBlockSize = 4; + const size_t kChunkSize = 6; + const size_t kShiftAmount = 3; + + const float kInput[kNumInputChannels][kNumFrames] = { + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}}; + ChannelBuffer input_cb(kNumFrames, kNumInputChannels); + input_cb.SetDataForTesting(kInput[0], sizeof(kInput) / sizeof(**kInput)); + + const float kExpectedOutput[kNumOutputChannels][kNumFrames] = { + {6, 10, 10, 20, 10, 10, 20, 10, 10, 20, 10, 10}, + {6, 14, 14, 28, 14, 14, 28, 14, 14, 28, 14, 14}}; + ChannelBuffer expected_output_cb(kNumFrames, kNumOutputChannels); + expected_output_cb.SetDataForTesting( + kExpectedOutput[0], sizeof(kExpectedOutput) / sizeof(**kExpectedOutput)); + + const float kWindow[kBlockSize] = {2.f, 2.f, 2.f, 2.f}; + + ChannelBuffer actual_output_cb(kNumFrames, kNumOutputChannels); + ChannelBuffer input_chunk_cb(kChunkSize, kNumInputChannels); + ChannelBuffer output_chunk_cb(kChunkSize, kNumOutputChannels); + + PlusThreeBlockerCallback callback; + Blocker blocker(kChunkSize, kBlockSize, kNumInputChannels, kNumOutputChannels, + kWindow, kShiftAmount, &callback); + + RunTest(&blocker, kChunkSize, kNumFrames, input_cb.channels(), + input_chunk_cb.channels(), actual_output_cb.channels(), + output_chunk_cb.channels(), kNumInputChannels, kNumOutputChannels); + + ValidateSignalEquality(expected_output_cb.channels(), + actual_output_cb.channels(), kNumOutputChannels, + kNumFrames); +} + +TEST_F(BlockerTest, TestBlockerNoOverlap) { + const size_t kNumInputChannels = 3; + const size_t kNumOutputChannels = 2; + const size_t kNumFrames = 12; + const size_t kBlockSize = 4; + const size_t kChunkSize = 4; + const size_t kShiftAmount = 4; + + const float kInput[kNumInputChannels][kNumFrames] = { + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}}; + ChannelBuffer input_cb(kNumFrames, kNumInputChannels); + input_cb.SetDataForTesting(kInput[0], sizeof(kInput) / sizeof(**kInput)); + + const float kExpectedOutput[kNumOutputChannels][kNumFrames] = { + {10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10}, + {14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14}}; + ChannelBuffer expected_output_cb(kNumFrames, kNumOutputChannels); + expected_output_cb.SetDataForTesting( + kExpectedOutput[0], sizeof(kExpectedOutput) / sizeof(**kExpectedOutput)); + + const float kWindow[kBlockSize] = {2.f, 2.f, 2.f, 2.f}; + + ChannelBuffer actual_output_cb(kNumFrames, kNumOutputChannels); + ChannelBuffer input_chunk_cb(kChunkSize, kNumInputChannels); + ChannelBuffer output_chunk_cb(kChunkSize, kNumOutputChannels); + + PlusThreeBlockerCallback callback; + Blocker blocker(kChunkSize, kBlockSize, kNumInputChannels, kNumOutputChannels, + kWindow, kShiftAmount, &callback); + + RunTest(&blocker, kChunkSize, kNumFrames, input_cb.channels(), + input_chunk_cb.channels(), actual_output_cb.channels(), + output_chunk_cb.channels(), kNumInputChannels, kNumOutputChannels); + + ValidateSignalEquality(expected_output_cb.channels(), + actual_output_cb.channels(), kNumOutputChannels, + kNumFrames); +} + +TEST_F(BlockerTest, InitialDelaysAreMinimum) { + const size_t kNumInputChannels = 3; + const size_t kNumOutputChannels = 2; + const size_t kNumFrames = 1280; + const size_t kChunkSize[] = {80, 80, 80, 80, 80, 80, + 160, 160, 160, 160, 160, 160}; + const size_t kBlockSize[] = {64, 64, 64, 128, 128, 128, + 128, 128, 128, 256, 256, 256}; + const size_t kShiftAmount[] = {16, 32, 64, 32, 64, 128, + 32, 64, 128, 64, 128, 256}; + const size_t kInitialDelay[] = {48, 48, 48, 112, 112, 112, + 96, 96, 96, 224, 224, 224}; + + float input[kNumInputChannels][kNumFrames]; + for (size_t i = 0; i < kNumInputChannels; ++i) { + for (size_t j = 0; j < kNumFrames; ++j) { + input[i][j] = i + 1; + } + } + ChannelBuffer input_cb(kNumFrames, kNumInputChannels); + input_cb.SetDataForTesting(input[0], sizeof(input) / sizeof(**input)); + + ChannelBuffer output_cb(kNumFrames, kNumOutputChannels); + + CopyBlockerCallback callback; + + for (size_t i = 0; i < arraysize(kChunkSize); ++i) { + std::unique_ptr window(new float[kBlockSize[i]]); + for (size_t j = 0; j < kBlockSize[i]; ++j) { + window[j] = 1.f; + } + + ChannelBuffer input_chunk_cb(kChunkSize[i], kNumInputChannels); + ChannelBuffer output_chunk_cb(kChunkSize[i], kNumOutputChannels); + + Blocker blocker(kChunkSize[i], kBlockSize[i], kNumInputChannels, + kNumOutputChannels, window.get(), kShiftAmount[i], + &callback); + + RunTest(&blocker, kChunkSize[i], kNumFrames, input_cb.channels(), + input_chunk_cb.channels(), output_cb.channels(), + output_chunk_cb.channels(), kNumInputChannels, kNumOutputChannels); + + ValidateInitialDelay(output_cb.channels(), kNumOutputChannels, kNumFrames, + kInitialDelay[i]); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform.cc new file mode 100644 index 0000000000..b1a6526bba --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform.cc @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/test/lapped_transform.h" + +#include +#include +#include + +#include "common_audio/real_fourier.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +void LappedTransform::BlockThunk::ProcessBlock(const float* const* input, + size_t num_frames, + size_t num_input_channels, + size_t num_output_channels, + float* const* output) { + RTC_CHECK_EQ(num_input_channels, parent_->num_in_channels_); + RTC_CHECK_EQ(num_output_channels, parent_->num_out_channels_); + RTC_CHECK_EQ(parent_->block_length_, num_frames); + + for (size_t i = 0; i < num_input_channels; ++i) { + memcpy(parent_->real_buf_.Row(i), input[i], num_frames * sizeof(*input[0])); + parent_->fft_->Forward(parent_->real_buf_.Row(i), + parent_->cplx_pre_.Row(i)); + } + + size_t block_length = + RealFourier::ComplexLength(RealFourier::FftOrder(num_frames)); + RTC_CHECK_EQ(parent_->cplx_length_, block_length); + parent_->block_processor_->ProcessAudioBlock( + parent_->cplx_pre_.Array(), num_input_channels, parent_->cplx_length_, + num_output_channels, parent_->cplx_post_.Array()); + + for (size_t i = 0; i < num_output_channels; ++i) { + parent_->fft_->Inverse(parent_->cplx_post_.Row(i), + parent_->real_buf_.Row(i)); + memcpy(output[i], parent_->real_buf_.Row(i), + num_frames * sizeof(*input[0])); + } +} + +LappedTransform::LappedTransform(size_t num_in_channels, + size_t num_out_channels, + size_t chunk_length, + const float* window, + size_t block_length, + size_t shift_amount, + Callback* callback) + : blocker_callback_(this), + num_in_channels_(num_in_channels), + num_out_channels_(num_out_channels), + block_length_(block_length), + chunk_length_(chunk_length), + block_processor_(callback), + blocker_(chunk_length_, + block_length_, + num_in_channels_, + num_out_channels_, + window, + shift_amount, + &blocker_callback_), + fft_(RealFourier::Create(RealFourier::FftOrder(block_length_))), + cplx_length_(RealFourier::ComplexLength(fft_->order())), + real_buf_(num_in_channels, + block_length_, + RealFourier::kFftBufferAlignment), + cplx_pre_(num_in_channels, + cplx_length_, + RealFourier::kFftBufferAlignment), + cplx_post_(num_out_channels, + cplx_length_, + RealFourier::kFftBufferAlignment) { + RTC_CHECK(num_in_channels_ > 0); + RTC_CHECK_GT(block_length_, 0); + RTC_CHECK_GT(chunk_length_, 0); + RTC_CHECK(block_processor_); + + // block_length_ power of 2? + RTC_CHECK_EQ(0, block_length_ & (block_length_ - 1)); +} + +LappedTransform::~LappedTransform() = default; + +void LappedTransform::ProcessChunk(const float* const* in_chunk, + float* const* out_chunk) { + blocker_.ProcessChunk(in_chunk, chunk_length_, num_in_channels_, + num_out_channels_, out_chunk); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform.h b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform.h new file mode 100644 index 0000000000..bb25c34a9e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform.h @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_OPUS_TEST_LAPPED_TRANSFORM_H_ +#define MODULES_AUDIO_CODING_CODECS_OPUS_TEST_LAPPED_TRANSFORM_H_ + +#include +#include + +#include "common_audio/real_fourier.h" +#include "modules/audio_coding/codecs/opus/test/blocker.h" +#include "rtc_base/memory/aligned_malloc.h" + +namespace webrtc { + +// Wrapper class for aligned arrays. Every row (and the first dimension) are +// aligned to the given byte alignment. +template +class AlignedArray { + public: + AlignedArray(size_t rows, size_t cols, size_t alignment) + : rows_(rows), cols_(cols) { + RTC_CHECK_GT(alignment, 0); + head_row_ = + static_cast(AlignedMalloc(rows_ * sizeof(*head_row_), alignment)); + for (size_t i = 0; i < rows_; ++i) { + head_row_[i] = static_cast( + AlignedMalloc(cols_ * sizeof(**head_row_), alignment)); + } + } + + ~AlignedArray() { + for (size_t i = 0; i < rows_; ++i) { + AlignedFree(head_row_[i]); + } + AlignedFree(head_row_); + } + + T* const* Array() { return head_row_; } + + const T* const* Array() const { return head_row_; } + + T* Row(size_t row) { + RTC_CHECK_LE(row, rows_); + return head_row_[row]; + } + + const T* Row(size_t row) const { + RTC_CHECK_LE(row, rows_); + return head_row_[row]; + } + + private: + size_t rows_; + size_t cols_; + T** head_row_; +}; + +// Helper class for audio processing modules which operate on frequency domain +// input derived from the windowed time domain audio stream. +// +// The input audio chunk is sliced into possibly overlapping blocks, multiplied +// by a window and transformed with an FFT implementation. The transformed data +// is supplied to the given callback for processing. The processed output is +// then inverse transformed into the time domain and spliced back into a chunk +// which constitutes the final output of this processing module. +class LappedTransform { + public: + class Callback { + public: + virtual ~Callback() {} + + virtual void ProcessAudioBlock(const std::complex* const* in_block, + size_t num_in_channels, + size_t frames, + size_t num_out_channels, + std::complex* const* out_block) = 0; + }; + + // Construct a transform instance. `chunk_length` is the number of samples in + // each channel. `window` defines the window, owned by the caller (a copy is + // made internally); `window` should have length equal to `block_length`. + // `block_length` defines the length of a block, in samples. + // `shift_amount` is in samples. `callback` is the caller-owned audio + // processing function called for each block of the input chunk. + LappedTransform(size_t num_in_channels, + size_t num_out_channels, + size_t chunk_length, + const float* window, + size_t block_length, + size_t shift_amount, + Callback* callback); + ~LappedTransform(); + + // Main audio processing helper method. Internally slices `in_chunk` into + // blocks, transforms them to frequency domain, calls the callback for each + // block and returns a de-blocked time domain chunk of audio through + // `out_chunk`. Both buffers are caller-owned. + void ProcessChunk(const float* const* in_chunk, float* const* out_chunk); + + // Get the chunk length. + // + // The chunk length is the number of samples per channel that must be passed + // to ProcessChunk via the parameter in_chunk. + // + // Returns the same chunk_length passed to the LappedTransform constructor. + size_t chunk_length() const { return chunk_length_; } + + // Get the number of input channels. + // + // This is the number of arrays that must be passed to ProcessChunk via + // in_chunk. + // + // Returns the same num_in_channels passed to the LappedTransform constructor. + size_t num_in_channels() const { return num_in_channels_; } + + // Get the number of output channels. + // + // This is the number of arrays that must be passed to ProcessChunk via + // out_chunk. + // + // Returns the same num_out_channels passed to the LappedTransform + // constructor. + size_t num_out_channels() const { return num_out_channels_; } + + // Returns the initial delay. + // + // This is the delay introduced by the `blocker_` to be able to get and return + // chunks of `chunk_length`, but process blocks of `block_length`. + size_t initial_delay() const { return blocker_.initial_delay(); } + + private: + // Internal middleware callback, given to the blocker. Transforms each block + // and hands it over to the processing method given at construction time. + class BlockThunk : public BlockerCallback { + public: + explicit BlockThunk(LappedTransform* parent) : parent_(parent) {} + + void ProcessBlock(const float* const* input, + size_t num_frames, + size_t num_input_channels, + size_t num_output_channels, + float* const* output) override; + + private: + LappedTransform* const parent_; + } blocker_callback_; + + const size_t num_in_channels_; + const size_t num_out_channels_; + + const size_t block_length_; + const size_t chunk_length_; + + Callback* const block_processor_; + Blocker blocker_; + + // TODO(alessiob): Replace RealFourier with a different FFT library. + std::unique_ptr fft_; + const size_t cplx_length_; + AlignedArray real_buf_; + AlignedArray > cplx_pre_; + AlignedArray > cplx_post_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_OPUS_TEST_LAPPED_TRANSFORM_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform_unittest.cc new file mode 100644 index 0000000000..1003ed52e5 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/test/lapped_transform_unittest.cc @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/opus/test/lapped_transform.h" + +#include +#include +#include + +#include "test/gtest.h" + +using std::complex; + +namespace { + +class NoopCallback : public webrtc::LappedTransform::Callback { + public: + NoopCallback() : block_num_(0) {} + + void ProcessAudioBlock(const complex* const* in_block, + size_t in_channels, + size_t frames, + size_t out_channels, + complex* const* out_block) override { + RTC_CHECK_EQ(in_channels, out_channels); + for (size_t i = 0; i < out_channels; ++i) { + memcpy(out_block[i], in_block[i], sizeof(**in_block) * frames); + } + ++block_num_; + } + + size_t block_num() { return block_num_; } + + private: + size_t block_num_; +}; + +class FftCheckerCallback : public webrtc::LappedTransform::Callback { + public: + FftCheckerCallback() : block_num_(0) {} + + void ProcessAudioBlock(const complex* const* in_block, + size_t in_channels, + size_t frames, + size_t out_channels, + complex* const* out_block) override { + RTC_CHECK_EQ(in_channels, out_channels); + + size_t full_length = (frames - 1) * 2; + ++block_num_; + + if (block_num_ > 0) { + ASSERT_NEAR(in_block[0][0].real(), static_cast(full_length), + 1e-5f); + ASSERT_NEAR(in_block[0][0].imag(), 0.0f, 1e-5f); + for (size_t i = 1; i < frames; ++i) { + ASSERT_NEAR(in_block[0][i].real(), 0.0f, 1e-5f); + ASSERT_NEAR(in_block[0][i].imag(), 0.0f, 1e-5f); + } + } + } + + size_t block_num() { return block_num_; } + + private: + size_t block_num_; +}; + +void SetFloatArray(float value, int rows, int cols, float* const* array) { + for (int i = 0; i < rows; ++i) { + for (int j = 0; j < cols; ++j) { + array[i][j] = value; + } + } +} + +} // namespace + +namespace webrtc { + +TEST(LappedTransformTest, Windowless) { + const size_t kChannels = 3; + const size_t kChunkLength = 512; + const size_t kBlockLength = 64; + const size_t kShiftAmount = 64; + NoopCallback noop; + + // Rectangular window. + float window[kBlockLength]; + std::fill(window, &window[kBlockLength], 1.0f); + + LappedTransform trans(kChannels, kChannels, kChunkLength, window, + kBlockLength, kShiftAmount, &noop); + float in_buffer[kChannels][kChunkLength]; + float* in_chunk[kChannels]; + float out_buffer[kChannels][kChunkLength]; + float* out_chunk[kChannels]; + + in_chunk[0] = in_buffer[0]; + in_chunk[1] = in_buffer[1]; + in_chunk[2] = in_buffer[2]; + out_chunk[0] = out_buffer[0]; + out_chunk[1] = out_buffer[1]; + out_chunk[2] = out_buffer[2]; + SetFloatArray(2.0f, kChannels, kChunkLength, in_chunk); + SetFloatArray(-1.0f, kChannels, kChunkLength, out_chunk); + + trans.ProcessChunk(in_chunk, out_chunk); + + for (size_t i = 0; i < kChannels; ++i) { + for (size_t j = 0; j < kChunkLength; ++j) { + ASSERT_NEAR(out_chunk[i][j], 2.0f, 1e-5f); + } + } + + ASSERT_EQ(kChunkLength / kBlockLength, noop.block_num()); +} + +TEST(LappedTransformTest, IdentityProcessor) { + const size_t kChunkLength = 512; + const size_t kBlockLength = 64; + const size_t kShiftAmount = 32; + NoopCallback noop; + + // Identity window for |overlap = block_size / 2|. + float window[kBlockLength]; + std::fill(window, &window[kBlockLength], std::sqrt(0.5f)); + + LappedTransform trans(1, 1, kChunkLength, window, kBlockLength, kShiftAmount, + &noop); + float in_buffer[kChunkLength]; + float* in_chunk = in_buffer; + float out_buffer[kChunkLength]; + float* out_chunk = out_buffer; + + SetFloatArray(2.0f, 1, kChunkLength, &in_chunk); + SetFloatArray(-1.0f, 1, kChunkLength, &out_chunk); + + trans.ProcessChunk(&in_chunk, &out_chunk); + + for (size_t i = 0; i < kChunkLength; ++i) { + ASSERT_NEAR(out_chunk[i], (i < kBlockLength - kShiftAmount) ? 0.0f : 2.0f, + 1e-5f); + } + + ASSERT_EQ(kChunkLength / kShiftAmount, noop.block_num()); +} + +TEST(LappedTransformTest, Callbacks) { + const size_t kChunkLength = 512; + const size_t kBlockLength = 64; + FftCheckerCallback call; + + // Rectangular window. + float window[kBlockLength]; + std::fill(window, &window[kBlockLength], 1.0f); + + LappedTransform trans(1, 1, kChunkLength, window, kBlockLength, kBlockLength, + &call); + float in_buffer[kChunkLength]; + float* in_chunk = in_buffer; + float out_buffer[kChunkLength]; + float* out_chunk = out_buffer; + + SetFloatArray(1.0f, 1, kChunkLength, &in_chunk); + SetFloatArray(-1.0f, 1, kChunkLength, &out_chunk); + + trans.ProcessChunk(&in_chunk, &out_chunk); + + ASSERT_EQ(kChunkLength / kBlockLength, call.block_num()); +} + +TEST(LappedTransformTest, chunk_length) { + const size_t kBlockLength = 64; + FftCheckerCallback call; + const float window[kBlockLength] = {}; + + // Make sure that chunk_length returns the same value passed to the + // LappedTransform constructor. + { + const size_t kExpectedChunkLength = 512; + const LappedTransform trans(1, 1, kExpectedChunkLength, window, + kBlockLength, kBlockLength, &call); + + EXPECT_EQ(kExpectedChunkLength, trans.chunk_length()); + } + { + const size_t kExpectedChunkLength = 160; + const LappedTransform trans(1, 1, kExpectedChunkLength, window, + kBlockLength, kBlockLength, &call); + + EXPECT_EQ(kExpectedChunkLength, trans.chunk_length()); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc new file mode 100644 index 0000000000..7761efe8b3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h" + +#include + +#include "modules/audio_coding/codecs/legacy_encoded_audio_frame.h" +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +AudioDecoderPcm16B::AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels) + : sample_rate_hz_(sample_rate_hz), num_channels_(num_channels) { + RTC_DCHECK(sample_rate_hz == 8000 || sample_rate_hz == 16000 || + sample_rate_hz == 32000 || sample_rate_hz == 48000) + << "Unsupported sample rate " << sample_rate_hz; + RTC_DCHECK_GE(num_channels, 1); +} + +void AudioDecoderPcm16B::Reset() {} + +int AudioDecoderPcm16B::SampleRateHz() const { + return sample_rate_hz_; +} + +size_t AudioDecoderPcm16B::Channels() const { + return num_channels_; +} + +int AudioDecoderPcm16B::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(sample_rate_hz_, sample_rate_hz); + // Adjust the encoded length down to ensure the same number of samples in each + // channel. + const size_t encoded_len_adjusted = + PacketDuration(encoded, encoded_len) * 2 * + Channels(); // 2 bytes per sample per channel + size_t ret = WebRtcPcm16b_Decode(encoded, encoded_len_adjusted, decoded); + *speech_type = ConvertSpeechType(1); + return static_cast(ret); +} + +std::vector AudioDecoderPcm16B::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + const int samples_per_ms = rtc::CheckedDivExact(sample_rate_hz_, 1000); + return LegacyEncodedAudioFrame::SplitBySamples( + this, std::move(payload), timestamp, samples_per_ms * 2 * num_channels_, + samples_per_ms); +} + +int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + // Two encoded byte per sample per channel. + return static_cast(encoded_len / (2 * Channels())); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h new file mode 100644 index 0000000000..6f50161d3f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_DECODER_PCM16B_H_ +#define MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_DECODER_PCM16B_H_ + +#include +#include + +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/buffer.h" + +namespace webrtc { + +class AudioDecoderPcm16B final : public AudioDecoder { + public: + AudioDecoderPcm16B(int sample_rate_hz, size_t num_channels); + + AudioDecoderPcm16B(const AudioDecoderPcm16B&) = delete; + AudioDecoderPcm16B& operator=(const AudioDecoderPcm16B&) = delete; + + void Reset() override; + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + int SampleRateHz() const override; + size_t Channels() const override; + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + private: + const int sample_rate_hz_; + const size_t num_channels_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_DECODER_PCM16B_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc new file mode 100644 index 0000000000..9445b1ee3e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h" + +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +size_t AudioEncoderPcm16B::EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) { + return WebRtcPcm16b_Encode(audio, input_len, encoded); +} + +size_t AudioEncoderPcm16B::BytesPerSample() const { + return 2; +} + +AudioEncoder::CodecType AudioEncoderPcm16B::GetCodecType() const { + return CodecType::kOther; +} + +bool AudioEncoderPcm16B::Config::IsOk() const { + if ((sample_rate_hz != 8000) && (sample_rate_hz != 16000) && + (sample_rate_hz != 32000) && (sample_rate_hz != 48000)) + return false; + return AudioEncoderPcm::Config::IsOk(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h new file mode 100644 index 0000000000..c363b40b3f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_ENCODER_PCM16B_H_ +#define MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_ENCODER_PCM16B_H_ + +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" + +namespace webrtc { + +class AudioEncoderPcm16B final : public AudioEncoderPcm { + public: + struct Config : public AudioEncoderPcm::Config { + public: + Config() : AudioEncoderPcm::Config(107), sample_rate_hz(8000) {} + bool IsOk() const; + + int sample_rate_hz; + }; + + explicit AudioEncoderPcm16B(const Config& config) + : AudioEncoderPcm(config, config.sample_rate_hz) {} + + AudioEncoderPcm16B(const AudioEncoderPcm16B&) = delete; + AudioEncoderPcm16B& operator=(const AudioEncoderPcm16B&) = delete; + + protected: + size_t EncodeCall(const int16_t* audio, + size_t input_len, + uint8_t* encoded) override; + + size_t BytesPerSample() const override; + + AudioEncoder::CodecType GetCodecType() const override; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_PCM16B_AUDIO_ENCODER_PCM16B_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c new file mode 100644 index 0000000000..2f6dce5f41 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" + +size_t WebRtcPcm16b_Encode(const int16_t* speech, + size_t len, + uint8_t* encoded) { + size_t i; + for (i = 0; i < len; ++i) { + uint16_t s = speech[i]; + encoded[2 * i] = s >> 8; + encoded[2 * i + 1] = s; + } + return 2 * len; +} + +size_t WebRtcPcm16b_Decode(const uint8_t* encoded, + size_t len, + int16_t* speech) { + size_t i; + for (i = 0; i < len / 2; ++i) + speech[i] = encoded[2 * i] << 8 | encoded[2 * i + 1]; + return len / 2; +} diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h new file mode 100644 index 0000000000..75d1efda3b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_H_ +#define MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_H_ +/* + * Define the fixpoint numeric formats + */ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * WebRtcPcm16b_Encode(...) + * + * "Encode" a sample vector to 16 bit linear (Encoded standard is big endian) + * + * Input: + * - speech : Input speech vector + * - len : Number of samples in speech vector + * + * Output: + * - encoded : Encoded data vector (big endian 16 bit) + * + * Returned value : Length (in bytes) of coded data. + * Always equal to twice the len input parameter. + */ + +size_t WebRtcPcm16b_Encode(const int16_t* speech, size_t len, uint8_t* encoded); + +/**************************************************************************** + * WebRtcPcm16b_Decode(...) + * + * "Decode" a vector to 16 bit linear (Encoded standard is big endian) + * + * Input: + * - encoded : Encoded data vector (big endian 16 bit) + * - len : Number of bytes in encoded + * + * Output: + * - speech : Decoded speech vector + * + * Returned value : Samples in speech + */ + +size_t WebRtcPcm16b_Decode(const uint8_t* encoded, size_t len, int16_t* speech); + +#ifdef __cplusplus +} +#endif + +#endif /* MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_H_ */ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc new file mode 100644 index 0000000000..ecf91b45ac --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/pcm16b/pcm16b_common.h" + +#include + +#include + +namespace webrtc { + +void Pcm16BAppendSupportedCodecSpecs(std::vector* specs) { + for (uint8_t num_channels : {1, 2}) { + for (int sample_rate_hz : {8000, 16000, 32000}) { + specs->push_back( + {{"L16", sample_rate_hz, num_channels}, + {sample_rate_hz, num_channels, sample_rate_hz * num_channels * 16}}); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.h b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.h new file mode 100644 index 0000000000..3fae717ff3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_COMMON_H_ +#define MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_COMMON_H_ + +#include + +#include "api/audio_codecs/audio_format.h" + +namespace webrtc { +void Pcm16BAppendSupportedCodecSpecs(std::vector* specs); +} + +#endif // MODULES_AUDIO_CODING_CODECS_PCM16B_PCM16B_COMMON_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc b/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc new file mode 100644 index 0000000000..724bba52d6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" + +#include + +#include +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/byte_order.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +static constexpr const int kRedMaxPacketSize = + 1 << 10; // RED packets must be less than 1024 bytes to fit the 10 bit + // block length. +static constexpr const size_t kRedMaxTimestampDelta = + 1 << 14; // RED packets can encode a timestamp delta of 14 bits. +static constexpr const size_t kAudioMaxRtpPacketLen = + 1200; // The typical MTU is 1200 bytes. + +static constexpr size_t kRedHeaderLength = 4; // 4 bytes RED header. +static constexpr size_t kRedLastHeaderLength = + 1; // reduced size for last RED header. + +static constexpr size_t kRedNumberOfRedundantEncodings = + 1; // The level of redundancy we support. + +AudioEncoderCopyRed::Config::Config() = default; +AudioEncoderCopyRed::Config::Config(Config&&) = default; +AudioEncoderCopyRed::Config::~Config() = default; + +size_t GetMaxRedundancyFromFieldTrial(const FieldTrialsView& field_trials) { + const std::string red_trial = + field_trials.Lookup("WebRTC-Audio-Red-For-Opus"); + size_t redundancy = 0; + if (sscanf(red_trial.c_str(), "Enabled-%zu", &redundancy) != 1 || + redundancy > 9) { + return kRedNumberOfRedundantEncodings; + } + return redundancy; +} + +AudioEncoderCopyRed::AudioEncoderCopyRed(Config&& config, + const FieldTrialsView& field_trials) + : speech_encoder_(std::move(config.speech_encoder)), + primary_encoded_(0, kAudioMaxRtpPacketLen), + max_packet_length_(kAudioMaxRtpPacketLen), + red_payload_type_(config.payload_type) { + RTC_CHECK(speech_encoder_) << "Speech encoder not provided."; + + auto number_of_redundant_encodings = + GetMaxRedundancyFromFieldTrial(field_trials); + for (size_t i = 0; i < number_of_redundant_encodings; i++) { + std::pair redundant; + redundant.second.EnsureCapacity(kAudioMaxRtpPacketLen); + redundant_encodings_.push_front(std::move(redundant)); + } +} + +AudioEncoderCopyRed::~AudioEncoderCopyRed() = default; + +int AudioEncoderCopyRed::SampleRateHz() const { + return speech_encoder_->SampleRateHz(); +} + +size_t AudioEncoderCopyRed::NumChannels() const { + return speech_encoder_->NumChannels(); +} + +int AudioEncoderCopyRed::RtpTimestampRateHz() const { + return speech_encoder_->RtpTimestampRateHz(); +} + +size_t AudioEncoderCopyRed::Num10MsFramesInNextPacket() const { + return speech_encoder_->Num10MsFramesInNextPacket(); +} + +size_t AudioEncoderCopyRed::Max10MsFramesInAPacket() const { + return speech_encoder_->Max10MsFramesInAPacket(); +} + +int AudioEncoderCopyRed::GetTargetBitrate() const { + return speech_encoder_->GetTargetBitrate(); +} + +AudioEncoder::EncodedInfo AudioEncoderCopyRed::EncodeImpl( + uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) { + primary_encoded_.Clear(); + EncodedInfo info = + speech_encoder_->Encode(rtp_timestamp, audio, &primary_encoded_); + RTC_CHECK(info.redundant.empty()) << "Cannot use nested redundant encoders."; + RTC_DCHECK_EQ(primary_encoded_.size(), info.encoded_bytes); + + if (info.encoded_bytes == 0 || info.encoded_bytes >= kRedMaxPacketSize) { + return info; + } + RTC_DCHECK_GT(max_packet_length_, info.encoded_bytes); + + size_t header_length_bytes = kRedLastHeaderLength; + size_t bytes_available = max_packet_length_ - info.encoded_bytes; + auto it = redundant_encodings_.begin(); + + // Determine how much redundancy we can fit into our packet by + // iterating forward. This is determined both by the length as well + // as the timestamp difference. The latter can occur with opus DTX which + // has timestamp gaps of 400ms which exceeds REDs timestamp delta field size. + for (; it != redundant_encodings_.end(); it++) { + if (bytes_available < kRedHeaderLength + it->first.encoded_bytes) { + break; + } + if (it->first.encoded_bytes == 0) { + break; + } + if (rtp_timestamp - it->first.encoded_timestamp >= kRedMaxTimestampDelta) { + break; + } + bytes_available -= kRedHeaderLength + it->first.encoded_bytes; + header_length_bytes += kRedHeaderLength; + } + + // Allocate room for RFC 2198 header. + encoded->SetSize(header_length_bytes); + + // Iterate backwards and append the data. + size_t header_offset = 0; + while (it-- != redundant_encodings_.begin()) { + encoded->AppendData(it->second); + + const uint32_t timestamp_delta = + info.encoded_timestamp - it->first.encoded_timestamp; + encoded->data()[header_offset] = it->first.payload_type | 0x80; + rtc::SetBE16(static_cast(encoded->data()) + header_offset + 1, + (timestamp_delta << 2) | (it->first.encoded_bytes >> 8)); + encoded->data()[header_offset + 3] = it->first.encoded_bytes & 0xff; + header_offset += kRedHeaderLength; + info.redundant.push_back(it->first); + } + + // `info` will be implicitly cast to an EncodedInfoLeaf struct, effectively + // discarding the (empty) vector of redundant information. This is + // intentional. + if (header_length_bytes > kRedHeaderLength) { + info.redundant.push_back(info); + RTC_DCHECK_EQ(info.speech, + info.redundant[info.redundant.size() - 1].speech); + } + + encoded->AppendData(primary_encoded_); + RTC_DCHECK_EQ(header_offset, header_length_bytes - 1); + encoded->data()[header_offset] = info.payload_type; + + // Shift the redundant encodings. + auto rit = redundant_encodings_.rbegin(); + for (auto next = std::next(rit); next != redundant_encodings_.rend(); + rit++, next = std::next(rit)) { + rit->first = next->first; + rit->second.SetData(next->second); + } + it = redundant_encodings_.begin(); + if (it != redundant_encodings_.end()) { + it->first = info; + it->second.SetData(primary_encoded_); + } + + // Update main EncodedInfo. + info.payload_type = red_payload_type_; + info.encoded_bytes = encoded->size(); + return info; +} + +void AudioEncoderCopyRed::Reset() { + speech_encoder_->Reset(); + auto number_of_redundant_encodings = redundant_encodings_.size(); + redundant_encodings_.clear(); + for (size_t i = 0; i < number_of_redundant_encodings; i++) { + std::pair redundant; + redundant.second.EnsureCapacity(kAudioMaxRtpPacketLen); + redundant_encodings_.push_front(std::move(redundant)); + } +} + +bool AudioEncoderCopyRed::SetFec(bool enable) { + return speech_encoder_->SetFec(enable); +} + +bool AudioEncoderCopyRed::SetDtx(bool enable) { + return speech_encoder_->SetDtx(enable); +} + +bool AudioEncoderCopyRed::GetDtx() const { + return speech_encoder_->GetDtx(); +} + +bool AudioEncoderCopyRed::SetApplication(Application application) { + return speech_encoder_->SetApplication(application); +} + +void AudioEncoderCopyRed::SetMaxPlaybackRate(int frequency_hz) { + speech_encoder_->SetMaxPlaybackRate(frequency_hz); +} + +bool AudioEncoderCopyRed::EnableAudioNetworkAdaptor( + const std::string& config_string, + RtcEventLog* event_log) { + return speech_encoder_->EnableAudioNetworkAdaptor(config_string, event_log); +} + +void AudioEncoderCopyRed::DisableAudioNetworkAdaptor() { + speech_encoder_->DisableAudioNetworkAdaptor(); +} + +void AudioEncoderCopyRed::OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + speech_encoder_->OnReceivedUplinkPacketLossFraction( + uplink_packet_loss_fraction); +} + +void AudioEncoderCopyRed::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional bwe_period_ms) { + speech_encoder_->OnReceivedUplinkBandwidth(target_audio_bitrate_bps, + bwe_period_ms); +} + +void AudioEncoderCopyRed::OnReceivedUplinkAllocation( + BitrateAllocationUpdate update) { + speech_encoder_->OnReceivedUplinkAllocation(update); +} + +absl::optional> +AudioEncoderCopyRed::GetFrameLengthRange() const { + return speech_encoder_->GetFrameLengthRange(); +} + +void AudioEncoderCopyRed::OnReceivedRtt(int rtt_ms) { + speech_encoder_->OnReceivedRtt(rtt_ms); +} + +void AudioEncoderCopyRed::OnReceivedOverhead(size_t overhead_bytes_per_packet) { + max_packet_length_ = kAudioMaxRtpPacketLen - overhead_bytes_per_packet; + return speech_encoder_->OnReceivedOverhead(overhead_bytes_per_packet); +} + +void AudioEncoderCopyRed::SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms) { + return speech_encoder_->SetReceiverFrameLengthRange(min_frame_length_ms, + max_frame_length_ms); +} + +ANAStats AudioEncoderCopyRed::GetANAStats() const { + return speech_encoder_->GetANAStats(); +} + +rtc::ArrayView> +AudioEncoderCopyRed::ReclaimContainedEncoders() { + return rtc::ArrayView>(&speech_encoder_, 1); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h b/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h new file mode 100644 index 0000000000..359b5eaa17 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_RED_AUDIO_ENCODER_COPY_RED_H_ +#define MODULES_AUDIO_CODING_CODECS_RED_AUDIO_ENCODER_COPY_RED_H_ + +#include +#include + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/field_trials_view.h" +#include "api/units/time_delta.h" +#include "rtc_base/buffer.h" + +namespace webrtc { + +// This class implements redundant audio coding as described in +// https://tools.ietf.org/html/rfc2198 +// The class object will have an underlying AudioEncoder object that performs +// the actual encodings. The current class will gather the N latest encodings +// from the underlying codec into one packet. Currently N is hard-coded to 2. + +class AudioEncoderCopyRed final : public AudioEncoder { + public: + struct Config { + Config(); + Config(Config&&); + ~Config(); + int payload_type; + std::unique_ptr speech_encoder; + }; + + AudioEncoderCopyRed(Config&& config, const FieldTrialsView& field_trials); + + ~AudioEncoderCopyRed() override; + + AudioEncoderCopyRed(const AudioEncoderCopyRed&) = delete; + AudioEncoderCopyRed& operator=(const AudioEncoderCopyRed&) = delete; + + int SampleRateHz() const override; + size_t NumChannels() const override; + int RtpTimestampRateHz() const override; + size_t Num10MsFramesInNextPacket() const override; + size_t Max10MsFramesInAPacket() const override; + int GetTargetBitrate() const override; + + void Reset() override; + bool SetFec(bool enable) override; + + bool SetDtx(bool enable) override; + bool GetDtx() const override; + + bool SetApplication(Application application) override; + void SetMaxPlaybackRate(int frequency_hz) override; + bool EnableAudioNetworkAdaptor(const std::string& config_string, + RtcEventLog* event_log) override; + void DisableAudioNetworkAdaptor() override; + void OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional bwe_period_ms) override; + void OnReceivedUplinkAllocation(BitrateAllocationUpdate update) override; + void OnReceivedRtt(int rtt_ms) override; + void OnReceivedOverhead(size_t overhead_bytes_per_packet) override; + void SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms) override; + ANAStats GetANAStats() const override; + absl::optional> GetFrameLengthRange() + const override; + rtc::ArrayView> ReclaimContainedEncoders() + override; + + protected: + EncodedInfo EncodeImpl(uint32_t rtp_timestamp, + rtc::ArrayView audio, + rtc::Buffer* encoded) override; + + private: + std::unique_ptr speech_encoder_; + rtc::Buffer primary_encoded_; + size_t max_packet_length_; + int red_payload_type_; + std::list> redundant_encodings_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_RED_AUDIO_ENCODER_COPY_RED_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc new file mode 100644 index 0000000000..795a996624 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc @@ -0,0 +1,641 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/mock_audio_encoder.h" +#include "test/scoped_key_value_config.h" +#include "test/testsupport/rtc_expect_death.h" + +using ::testing::_; +using ::testing::Eq; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::MockFunction; +using ::testing::Not; +using ::testing::Optional; +using ::testing::Return; +using ::testing::SetArgPointee; + +namespace webrtc { + +namespace { +static const size_t kMaxNumSamples = 48 * 10 * 2; // 10 ms @ 48 kHz stereo. +static const size_t kRedLastHeaderLength = + 1; // 1 byte RED header for the last element. +} + +class AudioEncoderCopyRedTest : public ::testing::Test { + protected: + AudioEncoderCopyRedTest() + : mock_encoder_(new MockAudioEncoder), + timestamp_(4711), + sample_rate_hz_(16000), + num_audio_samples_10ms(sample_rate_hz_ / 100), + red_payload_type_(200) { + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type_; + config.speech_encoder = std::unique_ptr(mock_encoder_); + red_.reset(new AudioEncoderCopyRed(std::move(config), field_trials_)); + memset(audio_, 0, sizeof(audio_)); + EXPECT_CALL(*mock_encoder_, NumChannels()).WillRepeatedly(Return(1U)); + EXPECT_CALL(*mock_encoder_, SampleRateHz()) + .WillRepeatedly(Return(sample_rate_hz_)); + } + + void TearDown() override { red_.reset(); } + + void Encode() { + ASSERT_TRUE(red_.get() != NULL); + encoded_.Clear(); + encoded_info_ = red_->Encode( + timestamp_, + rtc::ArrayView(audio_, num_audio_samples_10ms), + &encoded_); + timestamp_ += rtc::checked_cast(num_audio_samples_10ms); + } + + test::ScopedKeyValueConfig field_trials_; + MockAudioEncoder* mock_encoder_; + std::unique_ptr red_; + uint32_t timestamp_; + int16_t audio_[kMaxNumSamples]; + const int sample_rate_hz_; + size_t num_audio_samples_10ms; + rtc::Buffer encoded_; + AudioEncoder::EncodedInfo encoded_info_; + const int red_payload_type_; +}; + +TEST_F(AudioEncoderCopyRedTest, CreateAndDestroy) {} + +TEST_F(AudioEncoderCopyRedTest, CheckSampleRatePropagation) { + EXPECT_CALL(*mock_encoder_, SampleRateHz()).WillOnce(Return(17)); + EXPECT_EQ(17, red_->SampleRateHz()); +} + +TEST_F(AudioEncoderCopyRedTest, CheckNumChannelsPropagation) { + EXPECT_CALL(*mock_encoder_, NumChannels()).WillOnce(Return(17U)); + EXPECT_EQ(17U, red_->NumChannels()); +} + +TEST_F(AudioEncoderCopyRedTest, CheckFrameSizePropagation) { + EXPECT_CALL(*mock_encoder_, Num10MsFramesInNextPacket()) + .WillOnce(Return(17U)); + EXPECT_EQ(17U, red_->Num10MsFramesInNextPacket()); +} + +TEST_F(AudioEncoderCopyRedTest, CheckMaxFrameSizePropagation) { + EXPECT_CALL(*mock_encoder_, Max10MsFramesInAPacket()).WillOnce(Return(17U)); + EXPECT_EQ(17U, red_->Max10MsFramesInAPacket()); +} + +TEST_F(AudioEncoderCopyRedTest, CheckTargetAudioBitratePropagation) { + EXPECT_CALL(*mock_encoder_, + OnReceivedUplinkBandwidth(4711, absl::optional())); + red_->OnReceivedUplinkBandwidth(4711, absl::nullopt); +} + +TEST_F(AudioEncoderCopyRedTest, CheckPacketLossFractionPropagation) { + EXPECT_CALL(*mock_encoder_, OnReceivedUplinkPacketLossFraction(0.5)); + red_->OnReceivedUplinkPacketLossFraction(0.5); +} + +TEST_F(AudioEncoderCopyRedTest, CheckGetFrameLengthRangePropagation) { + auto expected_range = + std::make_pair(TimeDelta::Millis(20), TimeDelta::Millis(20)); + EXPECT_CALL(*mock_encoder_, GetFrameLengthRange()) + .WillRepeatedly(Return(absl::make_optional(expected_range))); + EXPECT_THAT(red_->GetFrameLengthRange(), Optional(Eq(expected_range))); +} + +// Checks that the an Encode() call is immediately propagated to the speech +// encoder. +TEST_F(AudioEncoderCopyRedTest, CheckImmediateEncode) { + // Interleaving the EXPECT_CALL sequence with expectations on the MockFunction + // check ensures that exactly one call to EncodeImpl happens in each + // Encode call. + InSequence s; + MockFunction check; + for (int i = 1; i <= 6; ++i) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillRepeatedly(Return(AudioEncoder::EncodedInfo())); + EXPECT_CALL(check, Call(i)); + Encode(); + check.Call(i); + } +} + +// Checks that no output is produced if the underlying codec doesn't emit any +// new data, even if the RED codec is loaded with a secondary encoding. +TEST_F(AudioEncoderCopyRedTest, CheckNoOutput) { + static const size_t kEncodedSize = 17; + static const size_t kHeaderLenBytes = 5; + { + InSequence s; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(kEncodedSize))) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(0))) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(kEncodedSize))); + } + + // Start with one Encode() call that will produce output. + Encode(); + // First call is a special case, since it does not include a secondary + // payload. + EXPECT_EQ(0u, encoded_info_.redundant.size()); + EXPECT_EQ(kEncodedSize + kRedLastHeaderLength, encoded_info_.encoded_bytes); + + // Next call to the speech encoder will not produce any output. + Encode(); + EXPECT_EQ(0u, encoded_info_.encoded_bytes); + + // Final call to the speech encoder will produce output. + Encode(); + EXPECT_EQ(2 * kEncodedSize + kHeaderLenBytes, encoded_info_.encoded_bytes); + ASSERT_EQ(2u, encoded_info_.redundant.size()); +} + +// Checks that the correct payload sizes are populated into the redundancy +// information for a redundancy level of 1. +TEST_F(AudioEncoderCopyRedTest, CheckPayloadSizes1) { + // Let the mock encoder return payload sizes 1, 2, 3, ..., 10 for the sequence + // of calls. + static const int kNumPackets = 10; + InSequence s; + for (int encode_size = 1; encode_size <= kNumPackets; ++encode_size) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(encode_size))); + } + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + EXPECT_EQ(0u, encoded_info_.redundant.size()); + EXPECT_EQ(kRedLastHeaderLength + 1u, encoded_info_.encoded_bytes); + + for (size_t i = 2; i <= kNumPackets; ++i) { + Encode(); + ASSERT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(i, encoded_info_.redundant[1].encoded_bytes); + EXPECT_EQ(i - 1, encoded_info_.redundant[0].encoded_bytes); + EXPECT_EQ(5 + i + (i - 1), encoded_info_.encoded_bytes); + } +} + +// Checks that the correct payload sizes are populated into the redundancy +// information for a redundancy level of 0. +TEST_F(AudioEncoderCopyRedTest, CheckPayloadSizes0) { + webrtc::test::ScopedKeyValueConfig field_trials( + field_trials_, "WebRTC-Audio-Red-For-Opus/Enabled-0/"); + // Recreate the RED encoder to take the new field trial setting into account. + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type_; + config.speech_encoder = std::move(red_->ReclaimContainedEncoders()[0]); + red_.reset(new AudioEncoderCopyRed(std::move(config), field_trials)); + + // Let the mock encoder return payload sizes 1, 2, 3, ..., 10 for the sequence + // of calls. + static const int kNumPackets = 10; + InSequence s; + for (int encode_size = 1; encode_size <= kNumPackets; ++encode_size) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(encode_size))); + } + + for (size_t i = 1; i <= kNumPackets; ++i) { + Encode(); + ASSERT_EQ(0u, encoded_info_.redundant.size()); + EXPECT_EQ(1 + i, encoded_info_.encoded_bytes); + } +} +// Checks that the correct payload sizes are populated into the redundancy +// information for a redundancy level of 2. +TEST_F(AudioEncoderCopyRedTest, CheckPayloadSizes2) { + webrtc::test::ScopedKeyValueConfig field_trials( + field_trials_, "WebRTC-Audio-Red-For-Opus/Enabled-2/"); + // Recreate the RED encoder to take the new field trial setting into account. + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type_; + config.speech_encoder = std::move(red_->ReclaimContainedEncoders()[0]); + red_.reset(new AudioEncoderCopyRed(std::move(config), field_trials)); + + // Let the mock encoder return payload sizes 1, 2, 3, ..., 10 for the sequence + // of calls. + static const int kNumPackets = 10; + InSequence s; + for (int encode_size = 1; encode_size <= kNumPackets; ++encode_size) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(encode_size))); + } + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + EXPECT_EQ(0u, encoded_info_.redundant.size()); + EXPECT_EQ(kRedLastHeaderLength + 1u, encoded_info_.encoded_bytes); + + // Second call is also special since it does not include a tertiary + // payload. + Encode(); + EXPECT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(8u, encoded_info_.encoded_bytes); + + for (size_t i = 3; i <= kNumPackets; ++i) { + Encode(); + ASSERT_EQ(3u, encoded_info_.redundant.size()); + EXPECT_EQ(i, encoded_info_.redundant[2].encoded_bytes); + EXPECT_EQ(i - 1, encoded_info_.redundant[1].encoded_bytes); + EXPECT_EQ(i - 2, encoded_info_.redundant[0].encoded_bytes); + EXPECT_EQ(9 + i + (i - 1) + (i - 2), encoded_info_.encoded_bytes); + } +} + +// Checks that the correct payload sizes are populated into the redundancy +// information for a redundancy level of 3. +TEST_F(AudioEncoderCopyRedTest, CheckPayloadSizes3) { + webrtc::test::ScopedKeyValueConfig field_trials( + field_trials_, "WebRTC-Audio-Red-For-Opus/Enabled-3/"); + // Recreate the RED encoder to take the new field trial setting into account. + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type_; + config.speech_encoder = std::move(red_->ReclaimContainedEncoders()[0]); + red_.reset(new AudioEncoderCopyRed(std::move(config), field_trials_)); + + // Let the mock encoder return payload sizes 1, 2, 3, ..., 10 for the sequence + // of calls. + static const int kNumPackets = 10; + InSequence s; + for (int encode_size = 1; encode_size <= kNumPackets; ++encode_size) { + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(encode_size))); + } + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + EXPECT_EQ(0u, encoded_info_.redundant.size()); + EXPECT_EQ(kRedLastHeaderLength + 1u, encoded_info_.encoded_bytes); + + // Second call is also special since it does not include a tertiary + // payload. + Encode(); + EXPECT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(8u, encoded_info_.encoded_bytes); + + // Third call is also special since it does not include a quaternary + // payload. + Encode(); + EXPECT_EQ(3u, encoded_info_.redundant.size()); + EXPECT_EQ(15u, encoded_info_.encoded_bytes); + + for (size_t i = 4; i <= kNumPackets; ++i) { + Encode(); + ASSERT_EQ(4u, encoded_info_.redundant.size()); + EXPECT_EQ(i, encoded_info_.redundant[3].encoded_bytes); + EXPECT_EQ(i - 1, encoded_info_.redundant[2].encoded_bytes); + EXPECT_EQ(i - 2, encoded_info_.redundant[1].encoded_bytes); + EXPECT_EQ(i - 3, encoded_info_.redundant[0].encoded_bytes); + EXPECT_EQ(13 + i + (i - 1) + (i - 2) + (i - 3), + encoded_info_.encoded_bytes); + } +} + +// Checks that the correct timestamps are returned. +TEST_F(AudioEncoderCopyRedTest, CheckTimestamps) { + uint32_t primary_timestamp = timestamp_; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 17; + info.encoded_timestamp = timestamp_; + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + EXPECT_EQ(primary_timestamp, encoded_info_.encoded_timestamp); + + uint32_t secondary_timestamp = primary_timestamp; + primary_timestamp = timestamp_; + info.encoded_timestamp = timestamp_; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + + Encode(); + ASSERT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(primary_timestamp, encoded_info_.redundant[1].encoded_timestamp); + EXPECT_EQ(secondary_timestamp, encoded_info_.redundant[0].encoded_timestamp); + EXPECT_EQ(primary_timestamp, encoded_info_.encoded_timestamp); +} + +// Checks that the primary and secondary payloads are written correctly. +TEST_F(AudioEncoderCopyRedTest, CheckPayloads) { + // Let the mock encoder write payloads with increasing values. The first + // payload will have values 0, 1, 2, ..., kPayloadLenBytes - 1. + static const size_t kPayloadLenBytes = 5; + static const size_t kHeaderLenBytes = 5; + uint8_t payload[kPayloadLenBytes]; + for (uint8_t i = 0; i < kPayloadLenBytes; ++i) { + payload[i] = i; + } + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillRepeatedly(Invoke(MockAudioEncoder::CopyEncoding(payload))); + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + EXPECT_EQ(kRedLastHeaderLength + kPayloadLenBytes, + encoded_info_.encoded_bytes); + for (size_t i = 0; i < kPayloadLenBytes; ++i) { + EXPECT_EQ(i, encoded_.data()[kRedLastHeaderLength + i]); + } + + for (int j = 0; j < 1; ++j) { + // Increment all values of the payload by 10. + for (size_t i = 0; i < kPayloadLenBytes; ++i) + payload[i] += 10; + + Encode(); + ASSERT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(kPayloadLenBytes, encoded_info_.redundant[0].encoded_bytes); + EXPECT_EQ(kPayloadLenBytes, encoded_info_.redundant[1].encoded_bytes); + for (size_t i = 0; i < kPayloadLenBytes; ++i) { + // Check secondary payload. + EXPECT_EQ(j * 10 + i, encoded_.data()[kHeaderLenBytes + i]); + + // Check primary payload. + EXPECT_EQ((j + 1) * 10 + i, + encoded_.data()[kHeaderLenBytes + i + kPayloadLenBytes]); + } + } +} + +// Checks correct propagation of payload type. +TEST_F(AudioEncoderCopyRedTest, CheckPayloadType) { + const int primary_payload_type = red_payload_type_ + 1; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 17; + info.payload_type = primary_payload_type; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + + // First call is a special case, since it does not include a secondary + // payload. + Encode(); + ASSERT_EQ(0u, encoded_info_.redundant.size()); + + const int secondary_payload_type = red_payload_type_ + 2; + info.payload_type = secondary_payload_type; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + + Encode(); + ASSERT_EQ(2u, encoded_info_.redundant.size()); + EXPECT_EQ(secondary_payload_type, encoded_info_.redundant[1].payload_type); + EXPECT_EQ(primary_payload_type, encoded_info_.redundant[0].payload_type); + EXPECT_EQ(red_payload_type_, encoded_info_.payload_type); +} + +TEST_F(AudioEncoderCopyRedTest, CheckRFC2198Header) { + const int primary_payload_type = red_payload_type_ + 1; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 10; + info.encoded_timestamp = timestamp_; + info.payload_type = primary_payload_type; + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); + info.encoded_timestamp = timestamp_; // update timestamp. + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); // Second call will produce a redundant encoding. + + EXPECT_EQ(encoded_.size(), + 5u + 2 * 10u); // header size + two encoded payloads. + EXPECT_EQ(encoded_[0], primary_payload_type | 0x80); + + uint32_t timestamp_delta = encoded_info_.encoded_timestamp - + encoded_info_.redundant[0].encoded_timestamp; + // Timestamp delta is encoded as a 14 bit value. + EXPECT_EQ(encoded_[1], timestamp_delta >> 6); + EXPECT_EQ(static_cast(encoded_[2] >> 2), timestamp_delta & 0x3f); + // Redundant length is encoded as 10 bit value. + EXPECT_EQ(encoded_[2] & 0x3u, encoded_info_.redundant[1].encoded_bytes >> 8); + EXPECT_EQ(encoded_[3], encoded_info_.redundant[1].encoded_bytes & 0xff); + EXPECT_EQ(encoded_[4], primary_payload_type); + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); // Third call will produce a redundant encoding with double + // redundancy. + + EXPECT_EQ(encoded_.size(), + 5u + 2 * 10u); // header size + two encoded payloads. + EXPECT_EQ(encoded_[0], primary_payload_type | 0x80); + + timestamp_delta = encoded_info_.encoded_timestamp - + encoded_info_.redundant[0].encoded_timestamp; + // Timestamp delta is encoded as a 14 bit value. + EXPECT_EQ(encoded_[1], timestamp_delta >> 6); + EXPECT_EQ(static_cast(encoded_[2] >> 2), timestamp_delta & 0x3f); + // Redundant length is encoded as 10 bit value. + EXPECT_EQ(encoded_[2] & 0x3u, encoded_info_.redundant[1].encoded_bytes >> 8); + EXPECT_EQ(encoded_[3], encoded_info_.redundant[1].encoded_bytes & 0xff); + + EXPECT_EQ(encoded_[4], primary_payload_type | 0x80); + timestamp_delta = encoded_info_.encoded_timestamp - + encoded_info_.redundant[1].encoded_timestamp; +} + +// Variant with a redundancy of 0. +TEST_F(AudioEncoderCopyRedTest, CheckRFC2198Header0) { + webrtc::test::ScopedKeyValueConfig field_trials( + field_trials_, "WebRTC-Audio-Red-For-Opus/Enabled-0/"); + // Recreate the RED encoder to take the new field trial setting into account. + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type_; + config.speech_encoder = std::move(red_->ReclaimContainedEncoders()[0]); + red_.reset(new AudioEncoderCopyRed(std::move(config), field_trials)); + + const int primary_payload_type = red_payload_type_ + 1; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 10; + info.encoded_timestamp = timestamp_; + info.payload_type = primary_payload_type; + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); + info.encoded_timestamp = timestamp_; // update timestamp. + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); // Second call will not produce a redundant encoding. + + EXPECT_EQ(encoded_.size(), + 1u + 1 * 10u); // header size + one encoded payloads. + EXPECT_EQ(encoded_[0], primary_payload_type); +} +// Variant with a redundancy of 2. +TEST_F(AudioEncoderCopyRedTest, CheckRFC2198Header2) { + webrtc::test::ScopedKeyValueConfig field_trials( + field_trials_, "WebRTC-Audio-Red-For-Opus/Enabled-2/"); + // Recreate the RED encoder to take the new field trial setting into account. + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type_; + config.speech_encoder = std::move(red_->ReclaimContainedEncoders()[0]); + red_.reset(new AudioEncoderCopyRed(std::move(config), field_trials)); + + const int primary_payload_type = red_payload_type_ + 1; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 10; + info.encoded_timestamp = timestamp_; + info.payload_type = primary_payload_type; + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); + info.encoded_timestamp = timestamp_; // update timestamp. + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); // Second call will produce a redundant encoding. + + EXPECT_EQ(encoded_.size(), + 5u + 2 * 10u); // header size + two encoded payloads. + EXPECT_EQ(encoded_[0], primary_payload_type | 0x80); + + uint32_t timestamp_delta = encoded_info_.encoded_timestamp - + encoded_info_.redundant[0].encoded_timestamp; + // Timestamp delta is encoded as a 14 bit value. + EXPECT_EQ(encoded_[1], timestamp_delta >> 6); + EXPECT_EQ(static_cast(encoded_[2] >> 2), timestamp_delta & 0x3f); + // Redundant length is encoded as 10 bit value. + EXPECT_EQ(encoded_[2] & 0x3u, encoded_info_.redundant[1].encoded_bytes >> 8); + EXPECT_EQ(encoded_[3], encoded_info_.redundant[1].encoded_bytes & 0xff); + EXPECT_EQ(encoded_[4], primary_payload_type); + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); // Third call will produce a redundant encoding with double + // redundancy. + + EXPECT_EQ(encoded_.size(), + 9u + 3 * 10u); // header size + three encoded payloads. + EXPECT_EQ(encoded_[0], primary_payload_type | 0x80); + + timestamp_delta = encoded_info_.encoded_timestamp - + encoded_info_.redundant[0].encoded_timestamp; + // Timestamp delta is encoded as a 14 bit value. + EXPECT_EQ(encoded_[1], timestamp_delta >> 6); + EXPECT_EQ(static_cast(encoded_[2] >> 2), timestamp_delta & 0x3f); + // Redundant length is encoded as 10 bit value. + EXPECT_EQ(encoded_[2] & 0x3u, encoded_info_.redundant[1].encoded_bytes >> 8); + EXPECT_EQ(encoded_[3], encoded_info_.redundant[1].encoded_bytes & 0xff); + + EXPECT_EQ(encoded_[4], primary_payload_type | 0x80); + timestamp_delta = encoded_info_.encoded_timestamp - + encoded_info_.redundant[1].encoded_timestamp; + // Timestamp delta is encoded as a 14 bit value. + EXPECT_EQ(encoded_[5], timestamp_delta >> 6); + EXPECT_EQ(static_cast(encoded_[6] >> 2), timestamp_delta & 0x3f); + // Redundant length is encoded as 10 bit value. + EXPECT_EQ(encoded_[6] & 0x3u, encoded_info_.redundant[1].encoded_bytes >> 8); + EXPECT_EQ(encoded_[7], encoded_info_.redundant[1].encoded_bytes & 0xff); + EXPECT_EQ(encoded_[8], primary_payload_type); +} + +TEST_F(AudioEncoderCopyRedTest, RespectsPayloadMTU) { + const int primary_payload_type = red_payload_type_ + 1; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 600; + info.encoded_timestamp = timestamp_; + info.payload_type = primary_payload_type; + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); + info.encoded_timestamp = timestamp_; // update timestamp. + info.encoded_bytes = 500; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); // Second call will produce a redundant encoding. + + EXPECT_EQ(encoded_.size(), 5u + 600u + 500u); + + info.encoded_timestamp = timestamp_; // update timestamp. + info.encoded_bytes = 400; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); // Third call will drop the oldest packet. + EXPECT_EQ(encoded_.size(), 5u + 500u + 400u); +} + +TEST_F(AudioEncoderCopyRedTest, LargeTimestampGap) { + const int primary_payload_type = red_payload_type_ + 1; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 100; + info.encoded_timestamp = timestamp_; + info.payload_type = primary_payload_type; + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); + // Update timestamp to simulate a 400ms gap like the one + // opus DTX causes. + timestamp_ += 19200; + info.encoded_timestamp = timestamp_; // update timestamp. + info.encoded_bytes = 200; + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); + + // The old packet will be dropped. + EXPECT_EQ(encoded_.size(), 1u + 200u); +} + +#if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// This test fixture tests various error conditions that makes the +// AudioEncoderCng die via CHECKs. +class AudioEncoderCopyRedDeathTest : public AudioEncoderCopyRedTest { + protected: + AudioEncoderCopyRedDeathTest() : AudioEncoderCopyRedTest() {} +}; + +TEST_F(AudioEncoderCopyRedDeathTest, WrongFrameSize) { + num_audio_samples_10ms *= 2; // 20 ms frame. + RTC_EXPECT_DEATH(Encode(), ""); + num_audio_samples_10ms = 0; // Zero samples. + RTC_EXPECT_DEATH(Encode(), ""); +} + +TEST_F(AudioEncoderCopyRedDeathTest, NullSpeechEncoder) { + test::ScopedKeyValueConfig field_trials; + AudioEncoderCopyRed* red = NULL; + AudioEncoderCopyRed::Config config; + config.speech_encoder = NULL; + RTC_EXPECT_DEATH( + red = new AudioEncoderCopyRed(std::move(config), field_trials), + "Speech encoder not provided."); + // The delete operation is needed to avoid leak reports from memcheck. + delete red; +} + +#endif // GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.cc b/third_party/libwebrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.cc new file mode 100644 index 0000000000..537e6fcede --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/tools/audio_codec_speed_test.h" + +#include "rtc_base/checks.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +using ::std::get; + +namespace webrtc { + +AudioCodecSpeedTest::AudioCodecSpeedTest(int block_duration_ms, + int input_sampling_khz, + int output_sampling_khz) + : block_duration_ms_(block_duration_ms), + input_sampling_khz_(input_sampling_khz), + output_sampling_khz_(output_sampling_khz), + input_length_sample_( + static_cast(block_duration_ms_ * input_sampling_khz_)), + output_length_sample_( + static_cast(block_duration_ms_ * output_sampling_khz_)), + data_pointer_(0), + loop_length_samples_(0), + max_bytes_(0), + encoded_bytes_(0), + encoding_time_ms_(0.0), + decoding_time_ms_(0.0), + out_file_(NULL) {} + +void AudioCodecSpeedTest::SetUp() { + channels_ = get<0>(GetParam()); + bit_rate_ = get<1>(GetParam()); + in_filename_ = test::ResourcePath(get<2>(GetParam()), get<3>(GetParam())); + save_out_data_ = get<4>(GetParam()); + + FILE* fp = fopen(in_filename_.c_str(), "rb"); + RTC_DCHECK(fp); + + // Obtain file size. + fseek(fp, 0, SEEK_END); + loop_length_samples_ = ftell(fp) / sizeof(int16_t); + rewind(fp); + + // Allocate memory to contain the whole file. + in_data_.reset( + new int16_t[loop_length_samples_ + input_length_sample_ * channels_]); + + data_pointer_ = 0; + + // Copy the file into the buffer. + ASSERT_EQ(fread(&in_data_[0], sizeof(int16_t), loop_length_samples_, fp), + loop_length_samples_); + fclose(fp); + + // Add an extra block length of samples to the end of the array, starting + // over again from the beginning of the array. This is done to simplify + // the reading process when reading over the end of the loop. + memcpy(&in_data_[loop_length_samples_], &in_data_[0], + input_length_sample_ * channels_ * sizeof(int16_t)); + + max_bytes_ = input_length_sample_ * channels_ * sizeof(int16_t); + out_data_.reset(new int16_t[output_length_sample_ * channels_]); + bit_stream_.reset(new uint8_t[max_bytes_]); + + if (save_out_data_) { + std::string out_filename = + ::testing::UnitTest::GetInstance()->current_test_info()->name(); + + // Erase '/' + size_t found; + while ((found = out_filename.find('/')) != std::string::npos) + out_filename.replace(found, 1, "_"); + + out_filename = test::OutputPath() + out_filename + ".pcm"; + + out_file_ = fopen(out_filename.c_str(), "wb"); + RTC_DCHECK(out_file_); + + printf("Output to be saved in %s.\n", out_filename.c_str()); + } +} + +void AudioCodecSpeedTest::TearDown() { + if (save_out_data_) { + fclose(out_file_); + } +} + +void AudioCodecSpeedTest::EncodeDecode(size_t audio_duration_sec) { + size_t time_now_ms = 0; + float time_ms; + + printf("Coding %d kHz-sampled %zu-channel audio at %d bps ...\n", + input_sampling_khz_, channels_, bit_rate_); + + while (time_now_ms < audio_duration_sec * 1000) { + // Encode & decode. + time_ms = EncodeABlock(&in_data_[data_pointer_], &bit_stream_[0], + max_bytes_, &encoded_bytes_); + encoding_time_ms_ += time_ms; + time_ms = DecodeABlock(&bit_stream_[0], encoded_bytes_, &out_data_[0]); + decoding_time_ms_ += time_ms; + if (save_out_data_) { + fwrite(&out_data_[0], sizeof(int16_t), output_length_sample_ * channels_, + out_file_); + } + data_pointer_ = (data_pointer_ + input_length_sample_ * channels_) % + loop_length_samples_; + time_now_ms += block_duration_ms_; + } + + printf("Encoding: %.2f%% real time,\nDecoding: %.2f%% real time.\n", + (encoding_time_ms_ / audio_duration_sec) / 10.0, + (decoding_time_ms_ / audio_duration_sec) / 10.0); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.h b/third_party/libwebrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.h new file mode 100644 index 0000000000..c5f1d7c259 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/codecs/tools/audio_codec_speed_test.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_TOOLS_AUDIO_CODEC_SPEED_TEST_H_ +#define MODULES_AUDIO_CODING_CODECS_TOOLS_AUDIO_CODEC_SPEED_TEST_H_ + +#include +#include + +#include "test/gtest.h" + +namespace webrtc { + +// Define coding parameter as +// . +typedef std::tuple coding_param; + +class AudioCodecSpeedTest : public ::testing::TestWithParam { + protected: + AudioCodecSpeedTest(int block_duration_ms, + int input_sampling_khz, + int output_sampling_khz); + virtual void SetUp(); + virtual void TearDown(); + + // EncodeABlock(...) does the following: + // 1. encodes a block of audio, saved in `in_data`, + // 2. save the bit stream to `bit_stream` of `max_bytes` bytes in size, + // 3. assign `encoded_bytes` with the length of the bit stream (in bytes), + // 4. return the cost of time (in millisecond) spent on actual encoding. + virtual float EncodeABlock(int16_t* in_data, + uint8_t* bit_stream, + size_t max_bytes, + size_t* encoded_bytes) = 0; + + // DecodeABlock(...) does the following: + // 1. decodes the bit stream in `bit_stream` with a length of `encoded_bytes` + // (in bytes), + // 2. save the decoded audio in `out_data`, + // 3. return the cost of time (in millisecond) spent on actual decoding. + virtual float DecodeABlock(const uint8_t* bit_stream, + size_t encoded_bytes, + int16_t* out_data) = 0; + + // Encoding and decode an audio of `audio_duration` (in seconds) and + // record the runtime for encoding and decoding separately. + void EncodeDecode(size_t audio_duration); + + int block_duration_ms_; + int input_sampling_khz_; + int output_sampling_khz_; + + // Number of samples-per-channel in a frame. + size_t input_length_sample_; + + // Expected output number of samples-per-channel in a frame. + size_t output_length_sample_; + + std::unique_ptr in_data_; + std::unique_ptr out_data_; + size_t data_pointer_; + size_t loop_length_samples_; + std::unique_ptr bit_stream_; + + // Maximum number of bytes in output bitstream for a frame of audio. + size_t max_bytes_; + + size_t encoded_bytes_; + float encoding_time_ms_; + float decoding_time_ms_; + FILE* out_file_; + + size_t channels_; + + // Bit rate is in bit-per-second. + int bit_rate_; + + std::string in_filename_; + + // Determines whether to save the output to file. + bool save_out_data_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_CODECS_TOOLS_AUDIO_CODEC_SPEED_TEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/default_neteq_factory_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/default_neteq_factory_gn/moz.build new file mode 100644 index 0000000000..206a4ae313 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/default_neteq_factory_gn/moz.build @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("default_neteq_factory_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/g3doc/index.md b/third_party/libwebrtc/modules/audio_coding/g3doc/index.md new file mode 100644 index 0000000000..467563e696 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/g3doc/index.md @@ -0,0 +1,32 @@ + + + +# The WebRTC Audio Coding Module + +WebRTC audio coding module can handle both audio sending and receiving. Folder +[`acm2`][acm2] contains implementations of the APIs. + +* Audio Sending Audio frames, each of which should always contain 10 ms worth + of data, are provided to the audio coding module through + [`Add10MsData()`][Add10MsData]. The audio coding module uses a provided + audio encoder to encoded audio frames and deliver the data to a + pre-registered audio packetization callback, which is supposed to wrap the + encoded audio into RTP packets and send them over a transport. Built-in + audio codecs are included the [`codecs`][codecs] folder. The + [audio network adaptor][ANA] provides an add-on functionality to an audio + encoder (currently limited to Opus) to make the audio encoder adaptive to + network conditions (bandwidth, packet loss rate, etc). + +* Audio Receiving Audio packets are provided to the audio coding module + through [`IncomingPacket()`][IncomingPacket], and are processed by an audio + jitter buffer ([NetEq][NetEq]), which includes decoding of the packets. + Audio decoders are provided by an audio decoder factory. Decoded audio + samples should be queried by calling [`PlayoutData10Ms()`][PlayoutData10Ms]. + +[acm2]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/acm2/;drc=854d59f7501aac9e9bccfa7b4d1f7f4db7842719 +[Add10MsData]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/include/audio_coding_module.h;l=136;drc=d82a02c837d33cdfd75121e40dcccd32515e42d6 +[codecs]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/codecs/;drc=883fea1548d58e0080f98d66fab2e0c744dfb556 +[ANA]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/audio_network_adaptor/;drc=1f99551775cd876c116d1d90cba94c8a4670d184 +[IncomingPacket]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/include/audio_coding_module.h;l=192;drc=d82a02c837d33cdfd75121e40dcccd32515e42d6 +[NetEq]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/neteq/;drc=213dc2cfc5f1b360b1c6fc51d393491f5de49d3d +[PlayoutData10Ms]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/include/audio_coding_module.h;l=216;drc=d82a02c837d33cdfd75121e40dcccd32515e42d6 diff --git a/third_party/libwebrtc/modules/audio_coding/g711_c_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/g711_c_gn/moz.build new file mode 100644 index 0000000000..8322910094 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/g711_c_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/g711/g711_interface.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("g711_c_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/g711_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/g711_gn/moz.build new file mode 100644 index 0000000000..3a21cddc41 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/g711_gn/moz.build @@ -0,0 +1,226 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_decoder_pcm.cc", + "/third_party/libwebrtc/modules/audio_coding/codecs/g711/audio_encoder_pcm.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("g711_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/g722_c_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/g722_c_gn/moz.build new file mode 100644 index 0000000000..dc05788b80 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/g722_c_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/g722/g722_interface.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("g722_c_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/g722_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/g722_gn/moz.build new file mode 100644 index 0000000000..910eef267f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/g722_gn/moz.build @@ -0,0 +1,226 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_decoder_g722.cc", + "/third_party/libwebrtc/modules/audio_coding/codecs/g722/audio_encoder_g722.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("g722_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/ilbc_c_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/ilbc_c_gn/moz.build new file mode 100644 index 0000000000..1ff9bdf07a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/ilbc_c_gn/moz.build @@ -0,0 +1,300 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/abs_quant_loop.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/augmented_cb_corr.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/bw_expand.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_construct.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_augmentation.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_mem_energy_calc.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_search_core.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/cb_update_best_index.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/chebyshev.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/comp_corr.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/constants.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/create_augmented_vec.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decode_residual.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/decoder_interpolate_lsf.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/do_plc.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/encode.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/energy_inverse.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enh_upsample.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/enhancer_interface.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/filtered_cb_vecs.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/frame_classify.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_dequant.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/gain_quant.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_cd_vec.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_lsp_poly.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/get_sync_seq.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_input.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/hp_output.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/ilbc.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_dec.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/index_conv_enc.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_decode.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/init_encode.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/interpolate_samples.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lpc_encode.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_check.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_dec.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_interpolate_to_poly_enc.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_lsp.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsf_to_poly.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/lsp_to_lsf.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/my_corr.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/nearest_neighbor.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/pack_bits.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsf.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/poly_to_lsp.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/refiner.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_interpolate_lsf.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lpc_analysis.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_dequant.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/simple_lsf_quant.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/smooth_out_data.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/sort_sq.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/split_vq.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_construct.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/state_search.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/swap_bytes.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/unpack_bits.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq3.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/vq4.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/window32_w32.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/xcorr_coef.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("ilbc_c_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/ilbc_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/ilbc_gn/moz.build new file mode 100644 index 0000000000..4a7279386e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/ilbc_gn/moz.build @@ -0,0 +1,233 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.cc", + "/third_party/libwebrtc/modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("ilbc_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/include/audio_coding_module.h b/third_party/libwebrtc/modules/audio_coding/include/audio_coding_module.h new file mode 100644 index 0000000000..8b518fb979 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/include/audio_coding_module.h @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_H_ +#define MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_H_ + +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/function_view.h" +#include "api/neteq/neteq.h" +#include "api/neteq/neteq_factory.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +// forward declarations +class AudioDecoder; +class AudioEncoder; +class AudioFrame; +struct RTPHeader; + +// Callback class used for sending data ready to be packetized +class AudioPacketizationCallback { + public: + virtual ~AudioPacketizationCallback() {} + + virtual int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + int64_t absolute_capture_timestamp_ms) { + // TODO(bugs.webrtc.org/10739): Deprecate the old SendData and make this one + // pure virtual. + return SendData(frame_type, payload_type, timestamp, payload_data, + payload_len_bytes); + } + virtual int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes) { + RTC_DCHECK_NOTREACHED() << "This method must be overridden, or not used."; + return -1; + } +}; + +class AudioCodingModule { + protected: + AudioCodingModule() {} + + public: + struct Config { + explicit Config( + rtc::scoped_refptr decoder_factory = nullptr); + Config(const Config&); + ~Config(); + + NetEq::Config neteq_config; + Clock* clock; + rtc::scoped_refptr decoder_factory; + NetEqFactory* neteq_factory = nullptr; + }; + + static AudioCodingModule* Create(const Config& config); + virtual ~AudioCodingModule() = default; + + /////////////////////////////////////////////////////////////////////////// + // Sender + // + + // `modifier` is called exactly once with one argument: a pointer to the + // unique_ptr that holds the current encoder (which is null if there is no + // current encoder). For the duration of the call, `modifier` has exclusive + // access to the unique_ptr; it may call the encoder, steal the encoder and + // replace it with another encoder or with nullptr, etc. + virtual void ModifyEncoder( + rtc::FunctionView*)> modifier) = 0; + + // Utility method for simply replacing the existing encoder with a new one. + void SetEncoder(std::unique_ptr new_encoder) { + ModifyEncoder([&](std::unique_ptr* encoder) { + *encoder = std::move(new_encoder); + }); + } + + // int32_t RegisterTransportCallback() + // Register a transport callback which will be called to deliver + // the encoded buffers whenever Process() is called and a + // bit-stream is ready. + // + // Input: + // -transport : pointer to the callback class + // transport->SendData() is called whenever + // Process() is called and bit-stream is ready + // to deliver. + // + // Return value: + // -1 if the transport callback could not be registered + // 0 if registration is successful. + // + virtual int32_t RegisterTransportCallback( + AudioPacketizationCallback* transport) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t Add10MsData() + // Add 10MS of raw (PCM) audio data and encode it. If the sampling + // frequency of the audio does not match the sampling frequency of the + // current encoder ACM will resample the audio. If an encoded packet was + // produced, it will be delivered via the callback object registered using + // RegisterTransportCallback, and the return value from this function will + // be the number of bytes encoded. + // + // Input: + // -audio_frame : the input audio frame, containing raw audio + // sampling frequency etc. + // + // Return value: + // >= 0 number of bytes encoded. + // -1 some error occurred. + // + virtual int32_t Add10MsData(const AudioFrame& audio_frame) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int SetPacketLossRate() + // Sets expected packet loss rate for encoding. Some encoders provide packet + // loss gnostic encoding to make stream less sensitive to packet losses, + // through e.g., FEC. No effects on codecs that do not provide such encoding. + // + // Input: + // -packet_loss_rate : expected packet loss rate (0 -- 100 inclusive). + // + // Return value + // -1 if failed to set packet loss rate, + // 0 if succeeded. + // + // This is only used in test code that rely on old ACM APIs. + // TODO(minyue): Remove it when possible. + virtual int SetPacketLossRate(int packet_loss_rate) = 0; + + /////////////////////////////////////////////////////////////////////////// + // Receiver + // + + /////////////////////////////////////////////////////////////////////////// + // int32_t InitializeReceiver() + // Any decoder-related state of ACM will be initialized to the + // same state when ACM is created. This will not interrupt or + // effect encoding functionality of ACM. ACM would lose all the + // decoding-related settings by calling this function. + // For instance, all registered codecs are deleted and have to be + // registered again. + // + // Return value: + // -1 if failed to initialize, + // 0 if succeeded. + // + virtual int32_t InitializeReceiver() = 0; + + // Replace any existing decoders with the given payload type -> decoder map. + virtual void SetReceiveCodecs( + const std::map& codecs) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t IncomingPacket() + // Call this function to insert a parsed RTP packet into ACM. + // + // Inputs: + // -incoming_payload : received payload. + // -payload_len_bytes : the length of payload in bytes. + // -rtp_info : the relevant information retrieved from RTP + // header. + // + // Return value: + // -1 if failed to push in the payload + // 0 if payload is successfully pushed in. + // + virtual int32_t IncomingPacket(const uint8_t* incoming_payload, + size_t payload_len_bytes, + const RTPHeader& rtp_header) = 0; + + /////////////////////////////////////////////////////////////////////////// + // int32_t PlayoutData10Ms( + // Get 10 milliseconds of raw audio data for playout, at the given sampling + // frequency. ACM will perform a resampling if required. + // + // Input: + // -desired_freq_hz : the desired sampling frequency, in Hertz, of the + // output audio. If set to -1, the function returns + // the audio at the current sampling frequency. + // + // Output: + // -audio_frame : output audio frame which contains raw audio data + // and other relevant parameters. + // -muted : if true, the sample data in audio_frame is not + // populated, and must be interpreted as all zero. + // + // Return value: + // -1 if the function fails, + // 0 if the function succeeds. + // + virtual int32_t PlayoutData10Ms(int32_t desired_freq_hz, + AudioFrame* audio_frame, + bool* muted) = 0; + + /////////////////////////////////////////////////////////////////////////// + // statistics + // + + /////////////////////////////////////////////////////////////////////////// + // int32_t GetNetworkStatistics() + // Get network statistics. Note that the internal statistics of NetEq are + // reset by this call. + // + // Input: + // -network_statistics : a structure that contains network statistics. + // + // Return value: + // -1 if failed to set the network statistics, + // 0 if statistics are set successfully. + // + virtual int32_t GetNetworkStatistics( + NetworkStatistics* network_statistics) = 0; + + virtual ANAStats GetANAStats() const = 0; + + virtual int GetTargetBitrate() const = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/include/audio_coding_module_typedefs.h b/third_party/libwebrtc/modules/audio_coding/include/audio_coding_module_typedefs.h new file mode 100644 index 0000000000..9d2fcfe22e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/include/audio_coding_module_typedefs.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_TYPEDEFS_H_ +#define MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_TYPEDEFS_H_ + +#include + +namespace webrtc { + +/////////////////////////////////////////////////////////////////////////// +// enum ACMVADMode +// An enumerator for aggressiveness of VAD +// -VADNormal : least aggressive mode. +// -VADLowBitrate : more aggressive than "VADNormal" to save on +// bit-rate. +// -VADAggr : an aggressive mode. +// -VADVeryAggr : the most agressive mode. +// +enum ACMVADMode { + VADNormal = 0, + VADLowBitrate = 1, + VADAggr = 2, + VADVeryAggr = 3 +}; + +enum class AudioFrameType { + kEmptyFrame = 0, + kAudioFrameSpeech = 1, + kAudioFrameCN = 2, +}; + +/////////////////////////////////////////////////////////////////////////// +// +// Enumeration of Opus mode for intended application. +// +// kVoip : optimized for voice signals. +// kAudio : optimized for non-voice signals like music. +// +enum OpusApplicationMode { + kVoip = 0, + kAudio = 1, +}; + +// Statistics for calls to AudioCodingModule::PlayoutData10Ms(). +struct AudioDecodingCallStats { + AudioDecodingCallStats() + : calls_to_silence_generator(0), + calls_to_neteq(0), + decoded_normal(0), + decoded_neteq_plc(0), + decoded_codec_plc(0), + decoded_cng(0), + decoded_plc_cng(0), + decoded_muted_output(0) {} + + int calls_to_silence_generator; // Number of calls where silence generated, + // and NetEq was disengaged from decoding. + int calls_to_neteq; // Number of calls to NetEq. + int decoded_normal; // Number of calls where audio RTP packet decoded. + int decoded_neteq_plc; // Number of calls resulted in NetEq PLC. + int decoded_codec_plc; // Number of calls resulted in codec PLC. + int decoded_cng; // Number of calls where comfort noise generated due to DTX. + int decoded_plc_cng; // Number of calls resulted where PLC faded to CNG. + int decoded_muted_output; // Number of calls returning a muted state output. +}; + +// NETEQ statistics. +struct NetworkStatistics { + // current jitter buffer size in ms + uint16_t currentBufferSize; + // preferred (optimal) buffer size in ms + uint16_t preferredBufferSize; + // adding extra delay due to "peaky jitter" + bool jitterPeaksFound; + // Stats below correspond to similarly-named fields in the WebRTC stats spec. + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats + uint64_t totalSamplesReceived; + uint64_t concealedSamples; + uint64_t silentConcealedSamples; + uint64_t concealmentEvents; + uint64_t jitterBufferDelayMs; + uint64_t jitterBufferTargetDelayMs; + uint64_t jitterBufferMinimumDelayMs; + uint64_t jitterBufferEmittedCount; + uint64_t insertedSamplesForDeceleration; + uint64_t removedSamplesForAcceleration; + uint64_t fecPacketsReceived; + uint64_t fecPacketsDiscarded; + // Stats below correspond to similarly-named fields in the WebRTC stats spec. + // https://w3c.github.io/webrtc-stats/#dom-rtcreceivedrtpstreamstats + uint64_t packetsDiscarded; + // Stats below DO NOT correspond directly to anything in the WebRTC stats + // fraction (of original stream) of synthesized audio inserted through + // expansion (in Q14) + uint16_t currentExpandRate; + // fraction (of original stream) of synthesized speech inserted through + // expansion (in Q14) + uint16_t currentSpeechExpandRate; + // fraction of synthesized speech inserted through pre-emptive expansion + // (in Q14) + uint16_t currentPreemptiveRate; + // fraction of data removed through acceleration (in Q14) + uint16_t currentAccelerateRate; + // fraction of data coming from secondary decoding (in Q14) + uint16_t currentSecondaryDecodedRate; + // Fraction of secondary data, including FEC and RED, that is discarded (in + // Q14). Discarding of secondary data can be caused by the reception of the + // primary data, obsoleting the secondary data. It can also be caused by early + // or late arrival of secondary data. + uint16_t currentSecondaryDiscardedRate; + // average packet waiting time in the jitter buffer (ms) + int meanWaitingTimeMs; + // max packet waiting time in the jitter buffer (ms) + int maxWaitingTimeMs; + // count of the number of buffer flushes + uint64_t packetBufferFlushes; + // number of samples expanded due to delayed packets + uint64_t delayedPacketOutageSamples; + // arrival delay of incoming packets + uint64_t relativePacketArrivalDelayMs; + // number of audio interruptions + int32_t interruptionCount; + // total duration of audio interruptions + int32_t totalInterruptionDurationMs; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_INCLUDE_AUDIO_CODING_MODULE_TYPEDEFS_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/isac_bwinfo_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/isac_bwinfo_gn/moz.build new file mode 100644 index 0000000000..bb9cdf4460 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/isac_bwinfo_gn/moz.build @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("isac_bwinfo_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/isac_vad_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/isac_vad_gn/moz.build new file mode 100644 index 0000000000..86f566663a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/isac_vad_gn/moz.build @@ -0,0 +1,220 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/isac_vad.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c", + "/third_party/libwebrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("isac_vad_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/legacy_encoded_audio_frame_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/legacy_encoded_audio_frame_gn/moz.build new file mode 100644 index 0000000000..b642ffbe95 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/legacy_encoded_audio_frame_gn/moz.build @@ -0,0 +1,225 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/legacy_encoded_audio_frame.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("legacy_encoded_audio_frame_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/accelerate.cc b/third_party/libwebrtc/modules/audio_coding/neteq/accelerate.cc new file mode 100644 index 0000000000..f4ef6cdccb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/accelerate.cc @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/accelerate.h" + + +#include "api/array_view.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" + +namespace webrtc { + +Accelerate::ReturnCodes Accelerate::Process(const int16_t* input, + size_t input_length, + bool fast_accelerate, + AudioMultiVector* output, + size_t* length_change_samples) { + // Input length must be (almost) 30 ms. + static const size_t k15ms = 120; // 15 ms = 120 samples at 8 kHz sample rate. + if (num_channels_ == 0 || + input_length / num_channels_ < (2 * k15ms - 1) * fs_mult_) { + // Length of input data too short to do accelerate. Simply move all data + // from input to output. + output->PushBackInterleaved( + rtc::ArrayView(input, input_length)); + return kError; + } + return TimeStretch::Process(input, input_length, fast_accelerate, output, + length_change_samples); +} + +void Accelerate::SetParametersForPassiveSpeech(size_t /*len*/, + int16_t* best_correlation, + size_t* /*peak_index*/) const { + // When the signal does not contain any active speech, the correlation does + // not matter. Simply set it to zero. + *best_correlation = 0; +} + +Accelerate::ReturnCodes Accelerate::CheckCriteriaAndStretch( + const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool fast_mode, + AudioMultiVector* output) const { + // Check for strong correlation or passive speech. + // Use 8192 (0.5 in Q14) in fast mode. + const int correlation_threshold = fast_mode ? 8192 : kCorrelationThreshold; + if ((best_correlation > correlation_threshold) || !active_speech) { + // Do accelerate operation by overlap add. + + // Pre-calculate common multiplication with `fs_mult_`. + // 120 corresponds to 15 ms. + size_t fs_mult_120 = fs_mult_ * 120; + + if (fast_mode) { + // Fit as many multiples of `peak_index` as possible in fs_mult_120. + // TODO(henrik.lundin) Consider finding multiple correlation peaks and + // pick the one with the longest correlation lag in this case. + peak_index = (fs_mult_120 / peak_index) * peak_index; + } + + RTC_DCHECK_GE(fs_mult_120, peak_index); // Should be handled in Process(). + // Copy first part; 0 to 15 ms. + output->PushBackInterleaved( + rtc::ArrayView(input, fs_mult_120 * num_channels_)); + // Copy the `peak_index` starting at 15 ms to `temp_vector`. + AudioMultiVector temp_vector(num_channels_); + temp_vector.PushBackInterleaved(rtc::ArrayView( + &input[fs_mult_120 * num_channels_], peak_index * num_channels_)); + // Cross-fade `temp_vector` onto the end of `output`. + output->CrossFade(temp_vector, peak_index); + // Copy the last unmodified part, 15 ms + pitch period until the end. + output->PushBackInterleaved(rtc::ArrayView( + &input[(fs_mult_120 + peak_index) * num_channels_], + input_length - (fs_mult_120 + peak_index) * num_channels_)); + + if (active_speech) { + return kSuccess; + } else { + return kSuccessLowEnergy; + } + } else { + // Accelerate not allowed. Simply move all data from decoded to outData. + output->PushBackInterleaved( + rtc::ArrayView(input, input_length)); + return kNoStretch; + } +} + +Accelerate* AccelerateFactory::Create( + int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise) const { + return new Accelerate(sample_rate_hz, num_channels, background_noise); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/accelerate.h b/third_party/libwebrtc/modules/audio_coding/neteq/accelerate.h new file mode 100644 index 0000000000..01fe874d54 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/accelerate.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_ACCELERATE_H_ +#define MODULES_AUDIO_CODING_NETEQ_ACCELERATE_H_ + +#include +#include + +#include "modules/audio_coding/neteq/time_stretch.h" + +namespace webrtc { + +class AudioMultiVector; +class BackgroundNoise; + +// This class implements the Accelerate operation. Most of the work is done +// in the base class TimeStretch, which is shared with the PreemptiveExpand +// operation. In the Accelerate class, the operations that are specific to +// Accelerate are implemented. +class Accelerate : public TimeStretch { + public: + Accelerate(int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise) + : TimeStretch(sample_rate_hz, num_channels, background_noise) {} + + Accelerate(const Accelerate&) = delete; + Accelerate& operator=(const Accelerate&) = delete; + + // This method performs the actual Accelerate operation. The samples are + // read from `input`, of length `input_length` elements, and are written to + // `output`. The number of samples removed through time-stretching is + // is provided in the output `length_change_samples`. The method returns + // the outcome of the operation as an enumerator value. If `fast_accelerate` + // is true, the algorithm will relax the requirements on finding strong + // correlations, and may remove multiple pitch periods if possible. + ReturnCodes Process(const int16_t* input, + size_t input_length, + bool fast_accelerate, + AudioMultiVector* output, + size_t* length_change_samples); + + protected: + // Sets the parameters `best_correlation` and `peak_index` to suitable + // values when the signal contains no active speech. + void SetParametersForPassiveSpeech(size_t len, + int16_t* best_correlation, + size_t* peak_index) const override; + + // Checks the criteria for performing the time-stretching operation and, + // if possible, performs the time-stretching. + ReturnCodes CheckCriteriaAndStretch(const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool fast_mode, + AudioMultiVector* output) const override; +}; + +struct AccelerateFactory { + AccelerateFactory() {} + virtual ~AccelerateFactory() {} + + virtual Accelerate* Create(int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise) const; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_ACCELERATE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc new file mode 100644 index 0000000000..fef3c3c1e4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc @@ -0,0 +1,526 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include +#include +#include +#include + +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h" +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" +#include "modules/audio_coding/codecs/g722/audio_decoder_g722.h" +#include "modules/audio_coding/codecs/g722/audio_encoder_g722.h" +#include "modules/audio_coding/codecs/ilbc/audio_decoder_ilbc.h" +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" +#include "modules/audio_coding/codecs/opus/audio_decoder_opus.h" +#include "modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.h" +#include "modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h" +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "rtc_base/system/arch.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +namespace { + +constexpr int kOverheadBytesPerPacket = 50; + +// The absolute difference between the input and output (the first channel) is +// compared vs `tolerance`. The parameter `delay` is used to correct for codec +// delays. +void CompareInputOutput(const std::vector& input, + const std::vector& output, + size_t num_samples, + size_t channels, + int tolerance, + int delay) { + ASSERT_LE(num_samples, input.size()); + ASSERT_LE(num_samples * channels, output.size()); + for (unsigned int n = 0; n < num_samples - delay; ++n) { + ASSERT_NEAR(input[n], output[channels * n + delay], tolerance) + << "Exit test on first diff; n = " << n; + } +} + +// The absolute difference between the first two channels in `output` is +// compared vs `tolerance`. +void CompareTwoChannels(const std::vector& output, + size_t samples_per_channel, + size_t channels, + int tolerance) { + ASSERT_GE(channels, 2u); + ASSERT_LE(samples_per_channel * channels, output.size()); + for (unsigned int n = 0; n < samples_per_channel; ++n) + ASSERT_NEAR(output[channels * n], output[channels * n + 1], tolerance) + << "Stereo samples differ."; +} + +// Calculates mean-squared error between input and output (the first channel). +// The parameter `delay` is used to correct for codec delays. +double MseInputOutput(const std::vector& input, + const std::vector& output, + size_t num_samples, + size_t channels, + int delay) { + RTC_DCHECK_LT(delay, static_cast(num_samples)); + RTC_DCHECK_LE(num_samples, input.size()); + RTC_DCHECK_LE(num_samples * channels, output.size()); + if (num_samples == 0) + return 0.0; + double squared_sum = 0.0; + for (unsigned int n = 0; n < num_samples - delay; ++n) { + squared_sum += (input[n] - output[channels * n + delay]) * + (input[n] - output[channels * n + delay]); + } + return squared_sum / (num_samples - delay); +} +} // namespace + +class AudioDecoderTest : public ::testing::Test { + protected: + AudioDecoderTest() + : input_audio_( + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000), + codec_input_rate_hz_(32000), // Legacy default value. + frame_size_(0), + data_length_(0), + channels_(1), + payload_type_(17), + decoder_(NULL) {} + + ~AudioDecoderTest() override {} + + void SetUp() override { + if (audio_encoder_) + codec_input_rate_hz_ = audio_encoder_->SampleRateHz(); + // Create arrays. + ASSERT_GT(data_length_, 0u) << "The test must set data_length_ > 0"; + } + + void TearDown() override { + delete decoder_; + decoder_ = NULL; + } + + virtual void InitEncoder() {} + + // TODO(henrik.lundin) Change return type to size_t once most/all overriding + // implementations are gone. + virtual int EncodeFrame(const int16_t* input, + size_t input_len_samples, + rtc::Buffer* output) { + AudioEncoder::EncodedInfo encoded_info; + const size_t samples_per_10ms = audio_encoder_->SampleRateHz() / 100; + RTC_CHECK_EQ(samples_per_10ms * audio_encoder_->Num10MsFramesInNextPacket(), + input_len_samples); + std::unique_ptr interleaved_input( + new int16_t[channels_ * samples_per_10ms]); + for (size_t i = 0; i < audio_encoder_->Num10MsFramesInNextPacket(); ++i) { + EXPECT_EQ(0u, encoded_info.encoded_bytes); + + // Duplicate the mono input signal to however many channels the test + // wants. + test::InputAudioFile::DuplicateInterleaved(input + i * samples_per_10ms, + samples_per_10ms, channels_, + interleaved_input.get()); + + encoded_info = + audio_encoder_->Encode(0, + rtc::ArrayView( + interleaved_input.get(), + audio_encoder_->NumChannels() * + audio_encoder_->SampleRateHz() / 100), + output); + } + EXPECT_EQ(payload_type_, encoded_info.payload_type); + return static_cast(encoded_info.encoded_bytes); + } + + // Encodes and decodes audio. The absolute difference between the input and + // output is compared vs `tolerance`, and the mean-squared error is compared + // with `mse`. The encoded stream should contain `expected_bytes`. For stereo + // audio, the absolute difference between the two channels is compared vs + // `channel_diff_tolerance`. + void EncodeDecodeTest(size_t expected_bytes, + int tolerance, + double mse, + int delay = 0, + int channel_diff_tolerance = 0) { + ASSERT_GE(tolerance, 0) << "Test must define a tolerance >= 0"; + ASSERT_GE(channel_diff_tolerance, 0) + << "Test must define a channel_diff_tolerance >= 0"; + size_t processed_samples = 0u; + size_t encoded_bytes = 0u; + InitEncoder(); + std::vector input; + std::vector decoded; + while (processed_samples + frame_size_ <= data_length_) { + // Extend input vector with `frame_size_`. + input.resize(input.size() + frame_size_, 0); + // Read from input file. + ASSERT_GE(input.size() - processed_samples, frame_size_); + ASSERT_TRUE(input_audio_.Read(frame_size_, codec_input_rate_hz_, + &input[processed_samples])); + rtc::Buffer encoded; + size_t enc_len = + EncodeFrame(&input[processed_samples], frame_size_, &encoded); + // Make sure that frame_size_ * channels_ samples are allocated and free. + decoded.resize((processed_samples + frame_size_) * channels_, 0); + + const std::vector parse_result = + decoder_->ParsePayload(std::move(encoded), /*timestamp=*/0); + RTC_CHECK_EQ(parse_result.size(), size_t{1}); + auto decode_result = parse_result[0].frame->Decode( + rtc::ArrayView(&decoded[processed_samples * channels_], + frame_size_ * channels_ * sizeof(int16_t))); + RTC_CHECK(decode_result.has_value()); + EXPECT_EQ(frame_size_ * channels_, decode_result->num_decoded_samples); + encoded_bytes += enc_len; + processed_samples += frame_size_; + } + // For some codecs it doesn't make sense to check expected number of bytes, + // since the number can vary for different platforms. Opus is such a codec. + // In this case expected_bytes is set to 0. + if (expected_bytes) { + EXPECT_EQ(expected_bytes, encoded_bytes); + } + CompareInputOutput(input, decoded, processed_samples, channels_, tolerance, + delay); + if (channels_ == 2) + CompareTwoChannels(decoded, processed_samples, channels_, + channel_diff_tolerance); + EXPECT_LE( + MseInputOutput(input, decoded, processed_samples, channels_, delay), + mse); + } + + // Encodes a payload and decodes it twice with decoder re-init before each + // decode. Verifies that the decoded result is the same. + void ReInitTest() { + InitEncoder(); + std::unique_ptr input(new int16_t[frame_size_]); + ASSERT_TRUE( + input_audio_.Read(frame_size_, codec_input_rate_hz_, input.get())); + std::array encoded; + EncodeFrame(input.get(), frame_size_, &encoded[0]); + // Make a copy. + encoded[1].SetData(encoded[0].data(), encoded[0].size()); + + std::array, 2> outputs; + for (size_t i = 0; i < outputs.size(); ++i) { + outputs[i].resize(frame_size_ * channels_); + decoder_->Reset(); + const std::vector parse_result = + decoder_->ParsePayload(std::move(encoded[i]), /*timestamp=*/0); + RTC_CHECK_EQ(parse_result.size(), size_t{1}); + auto decode_result = parse_result[0].frame->Decode(outputs[i]); + RTC_CHECK(decode_result.has_value()); + EXPECT_EQ(frame_size_ * channels_, decode_result->num_decoded_samples); + } + EXPECT_EQ(outputs[0], outputs[1]); + } + + // Call DecodePlc and verify that the correct number of samples is produced. + void DecodePlcTest() { + InitEncoder(); + std::unique_ptr input(new int16_t[frame_size_]); + ASSERT_TRUE( + input_audio_.Read(frame_size_, codec_input_rate_hz_, input.get())); + rtc::Buffer encoded; + EncodeFrame(input.get(), frame_size_, &encoded); + decoder_->Reset(); + std::vector output(frame_size_ * channels_); + const std::vector parse_result = + decoder_->ParsePayload(std::move(encoded), /*timestamp=*/0); + RTC_CHECK_EQ(parse_result.size(), size_t{1}); + auto decode_result = parse_result[0].frame->Decode(output); + RTC_CHECK(decode_result.has_value()); + EXPECT_EQ(frame_size_ * channels_, decode_result->num_decoded_samples); + // Call DecodePlc and verify that we get one frame of data. + // (Overwrite the output from the above Decode call, but that does not + // matter.) + size_t dec_len = + decoder_->DecodePlc(/*num_frames=*/1, /*decoded=*/output.data()); + EXPECT_EQ(frame_size_ * channels_, dec_len); + } + + test::ResampleInputAudioFile input_audio_; + int codec_input_rate_hz_; + size_t frame_size_; + size_t data_length_; + size_t channels_; + const int payload_type_; + AudioDecoder* decoder_; + std::unique_ptr audio_encoder_; +}; + +class AudioDecoderPcmUTest : public AudioDecoderTest { + protected: + AudioDecoderPcmUTest() : AudioDecoderTest() { + frame_size_ = 160; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderPcmU(1); + AudioEncoderPcmU::Config config; + config.frame_size_ms = static_cast(frame_size_ / 8); + config.payload_type = payload_type_; + audio_encoder_.reset(new AudioEncoderPcmU(config)); + } +}; + +class AudioDecoderPcmATest : public AudioDecoderTest { + protected: + AudioDecoderPcmATest() : AudioDecoderTest() { + frame_size_ = 160; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderPcmA(1); + AudioEncoderPcmA::Config config; + config.frame_size_ms = static_cast(frame_size_ / 8); + config.payload_type = payload_type_; + audio_encoder_.reset(new AudioEncoderPcmA(config)); + } +}; + +class AudioDecoderPcm16BTest : public AudioDecoderTest { + protected: + AudioDecoderPcm16BTest() : AudioDecoderTest() { + codec_input_rate_hz_ = 16000; + frame_size_ = 20 * codec_input_rate_hz_ / 1000; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderPcm16B(codec_input_rate_hz_, 1); + RTC_DCHECK(decoder_); + AudioEncoderPcm16B::Config config; + config.sample_rate_hz = codec_input_rate_hz_; + config.frame_size_ms = + static_cast(frame_size_ / (config.sample_rate_hz / 1000)); + config.payload_type = payload_type_; + audio_encoder_.reset(new AudioEncoderPcm16B(config)); + } +}; + +class AudioDecoderIlbcTest : public AudioDecoderTest { + protected: + AudioDecoderIlbcTest() : AudioDecoderTest() { + codec_input_rate_hz_ = 8000; + frame_size_ = 240; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderIlbcImpl; + RTC_DCHECK(decoder_); + AudioEncoderIlbcConfig config; + config.frame_size_ms = 30; + audio_encoder_.reset(new AudioEncoderIlbcImpl(config, payload_type_)); + } + + // Overload the default test since iLBC's function WebRtcIlbcfix_NetEqPlc does + // not return any data. It simply resets a few states and returns 0. + void DecodePlcTest() { + InitEncoder(); + std::unique_ptr input(new int16_t[frame_size_]); + ASSERT_TRUE( + input_audio_.Read(frame_size_, codec_input_rate_hz_, input.get())); + rtc::Buffer encoded; + size_t enc_len = EncodeFrame(input.get(), frame_size_, &encoded); + AudioDecoder::SpeechType speech_type; + decoder_->Reset(); + std::unique_ptr output(new int16_t[frame_size_ * channels_]); + size_t dec_len = decoder_->Decode( + encoded.data(), enc_len, codec_input_rate_hz_, + frame_size_ * channels_ * sizeof(int16_t), output.get(), &speech_type); + EXPECT_EQ(frame_size_, dec_len); + // Simply call DecodePlc and verify that we get 0 as return value. + EXPECT_EQ(0U, decoder_->DecodePlc(1, output.get())); + } +}; + +class AudioDecoderG722Test : public AudioDecoderTest { + protected: + AudioDecoderG722Test() : AudioDecoderTest() { + codec_input_rate_hz_ = 16000; + frame_size_ = 160; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderG722Impl; + RTC_DCHECK(decoder_); + AudioEncoderG722Config config; + config.frame_size_ms = 10; + config.num_channels = 1; + audio_encoder_.reset(new AudioEncoderG722Impl(config, payload_type_)); + } +}; + +class AudioDecoderG722StereoTest : public AudioDecoderTest { + protected: + AudioDecoderG722StereoTest() : AudioDecoderTest() { + channels_ = 2; + codec_input_rate_hz_ = 16000; + frame_size_ = 160; + data_length_ = 10 * frame_size_; + decoder_ = new AudioDecoderG722StereoImpl; + RTC_DCHECK(decoder_); + AudioEncoderG722Config config; + config.frame_size_ms = 10; + config.num_channels = 2; + audio_encoder_.reset(new AudioEncoderG722Impl(config, payload_type_)); + } +}; + +class AudioDecoderOpusTest + : public AudioDecoderTest, + public testing::WithParamInterface> { + protected: + AudioDecoderOpusTest() : AudioDecoderTest() { + channels_ = opus_num_channels_; + codec_input_rate_hz_ = opus_sample_rate_hz_; + frame_size_ = rtc::CheckedDivExact(opus_sample_rate_hz_, 100); + data_length_ = 10 * frame_size_; + decoder_ = + new AudioDecoderOpusImpl(opus_num_channels_, opus_sample_rate_hz_); + AudioEncoderOpusConfig config; + config.frame_size_ms = 10; + config.sample_rate_hz = opus_sample_rate_hz_; + config.num_channels = opus_num_channels_; + config.application = opus_num_channels_ == 1 + ? AudioEncoderOpusConfig::ApplicationMode::kVoip + : AudioEncoderOpusConfig::ApplicationMode::kAudio; + audio_encoder_ = AudioEncoderOpus::MakeAudioEncoder(config, payload_type_); + audio_encoder_->OnReceivedOverhead(kOverheadBytesPerPacket); + } + const int opus_sample_rate_hz_{std::get<0>(GetParam())}; + const int opus_num_channels_{std::get<1>(GetParam())}; +}; + +INSTANTIATE_TEST_SUITE_P(Param, + AudioDecoderOpusTest, + testing::Combine(testing::Values(16000, 48000), + testing::Values(1, 2))); + +TEST_F(AudioDecoderPcmUTest, EncodeDecode) { + int tolerance = 251; + double mse = 1734.0; + EncodeDecodeTest(data_length_, tolerance, mse); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +namespace { +int SetAndGetTargetBitrate(AudioEncoder* audio_encoder, int rate) { + audio_encoder->OnReceivedUplinkBandwidth(rate, absl::nullopt); + return audio_encoder->GetTargetBitrate(); +} +void TestSetAndGetTargetBitratesWithFixedCodec(AudioEncoder* audio_encoder, + int fixed_rate) { + EXPECT_EQ(fixed_rate, SetAndGetTargetBitrate(audio_encoder, 32000)); + EXPECT_EQ(fixed_rate, SetAndGetTargetBitrate(audio_encoder, fixed_rate - 1)); + EXPECT_EQ(fixed_rate, SetAndGetTargetBitrate(audio_encoder, fixed_rate)); + EXPECT_EQ(fixed_rate, SetAndGetTargetBitrate(audio_encoder, fixed_rate + 1)); +} +} // namespace + +TEST_F(AudioDecoderPcmUTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 64000); +} + +TEST_F(AudioDecoderPcmATest, EncodeDecode) { + int tolerance = 308; + double mse = 1931.0; + EncodeDecodeTest(data_length_, tolerance, mse); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderPcmATest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 64000); +} + +TEST_F(AudioDecoderPcm16BTest, EncodeDecode) { + int tolerance = 0; + double mse = 0.0; + EncodeDecodeTest(2 * data_length_, tolerance, mse); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderPcm16BTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), + codec_input_rate_hz_ * 16); +} + +TEST_F(AudioDecoderIlbcTest, EncodeDecode) { + int tolerance = 6808; + double mse = 2.13e6; + int delay = 80; // Delay from input to output. + EncodeDecodeTest(500, tolerance, mse, delay); + ReInitTest(); + EXPECT_TRUE(decoder_->HasDecodePlc()); + DecodePlcTest(); +} + +TEST_F(AudioDecoderIlbcTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 13333); +} + +TEST_F(AudioDecoderG722Test, EncodeDecode) { + int tolerance = 6176; + double mse = 238630.0; + int delay = 22; // Delay from input to output. + EncodeDecodeTest(data_length_ / 2, tolerance, mse, delay); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderG722Test, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 64000); +} + +TEST_F(AudioDecoderG722StereoTest, EncodeDecode) { + int tolerance = 6176; + int channel_diff_tolerance = 0; + double mse = 238630.0; + int delay = 22; // Delay from input to output. + EncodeDecodeTest(data_length_, tolerance, mse, delay, channel_diff_tolerance); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_F(AudioDecoderG722StereoTest, SetTargetBitrate) { + TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 128000); +} + +// TODO(http://bugs.webrtc.org/12518): Enable the test after Opus has been +// updated. +TEST_P(AudioDecoderOpusTest, DISABLED_EncodeDecode) { + constexpr int tolerance = 6176; + constexpr int channel_diff_tolerance = 6; + constexpr double mse = 238630.0; + constexpr int delay = 22; // Delay from input to output. + EncodeDecodeTest(0, tolerance, mse, delay, channel_diff_tolerance); + ReInitTest(); + EXPECT_FALSE(decoder_->HasDecodePlc()); +} + +TEST_P(AudioDecoderOpusTest, SetTargetBitrate) { + const int overhead_rate = + 8 * kOverheadBytesPerPacket * codec_input_rate_hz_ / frame_size_; + EXPECT_EQ(6000, + SetAndGetTargetBitrate(audio_encoder_.get(), 5999 + overhead_rate)); + EXPECT_EQ(6000, + SetAndGetTargetBitrate(audio_encoder_.get(), 6000 + overhead_rate)); + EXPECT_EQ(32000, SetAndGetTargetBitrate(audio_encoder_.get(), + 32000 + overhead_rate)); + EXPECT_EQ(510000, SetAndGetTargetBitrate(audio_encoder_.get(), + 510000 + overhead_rate)); + EXPECT_EQ(510000, SetAndGetTargetBitrate(audio_encoder_.get(), + 511000 + overhead_rate)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.cc b/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.cc new file mode 100644 index 0000000000..14ae94649b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.cc @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/audio_multi_vector.h" + + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +AudioMultiVector::AudioMultiVector(size_t N) { + RTC_DCHECK_GT(N, 0); + if (N < 1) + N = 1; + for (size_t n = 0; n < N; ++n) { + channels_.push_back(new AudioVector); + } + num_channels_ = N; +} + +AudioMultiVector::AudioMultiVector(size_t N, size_t initial_size) { + RTC_DCHECK_GT(N, 0); + if (N < 1) + N = 1; + for (size_t n = 0; n < N; ++n) { + channels_.push_back(new AudioVector(initial_size)); + } + num_channels_ = N; +} + +AudioMultiVector::~AudioMultiVector() { + std::vector::iterator it = channels_.begin(); + while (it != channels_.end()) { + delete (*it); + ++it; + } +} + +void AudioMultiVector::Clear() { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->Clear(); + } +} + +void AudioMultiVector::Zeros(size_t length) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->Clear(); + channels_[i]->Extend(length); + } +} + +void AudioMultiVector::CopyTo(AudioMultiVector* copy_to) const { + if (copy_to) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->CopyTo(&(*copy_to)[i]); + } + } +} + +void AudioMultiVector::PushBackInterleaved( + rtc::ArrayView append_this) { + RTC_DCHECK_EQ(append_this.size() % num_channels_, 0); + if (append_this.empty()) { + return; + } + if (num_channels_ == 1) { + // Special case to avoid extra allocation and data shuffling. + channels_[0]->PushBack(append_this.data(), append_this.size()); + return; + } + size_t length_per_channel = append_this.size() / num_channels_; + int16_t* temp_array = new int16_t[length_per_channel]; // Temporary storage. + for (size_t channel = 0; channel < num_channels_; ++channel) { + // Copy elements to `temp_array`. + for (size_t i = 0; i < length_per_channel; ++i) { + temp_array[i] = append_this[channel + i * num_channels_]; + } + channels_[channel]->PushBack(temp_array, length_per_channel); + } + delete[] temp_array; +} + +void AudioMultiVector::PushBack(const AudioMultiVector& append_this) { + RTC_DCHECK_EQ(num_channels_, append_this.num_channels_); + if (num_channels_ == append_this.num_channels_) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->PushBack(append_this[i]); + } + } +} + +void AudioMultiVector::PushBackFromIndex(const AudioMultiVector& append_this, + size_t index) { + RTC_DCHECK_LT(index, append_this.Size()); + index = std::min(index, append_this.Size() - 1); + size_t length = append_this.Size() - index; + RTC_DCHECK_EQ(num_channels_, append_this.num_channels_); + if (num_channels_ == append_this.num_channels_) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->PushBack(append_this[i], length, index); + } + } +} + +void AudioMultiVector::PopFront(size_t length) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->PopFront(length); + } +} + +void AudioMultiVector::PopBack(size_t length) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->PopBack(length); + } +} + +size_t AudioMultiVector::ReadInterleaved(size_t length, + int16_t* destination) const { + return ReadInterleavedFromIndex(0, length, destination); +} + +size_t AudioMultiVector::ReadInterleavedFromIndex(size_t start_index, + size_t length, + int16_t* destination) const { + RTC_DCHECK(destination); + size_t index = 0; // Number of elements written to `destination` so far. + RTC_DCHECK_LE(start_index, Size()); + start_index = std::min(start_index, Size()); + if (length + start_index > Size()) { + length = Size() - start_index; + } + if (num_channels_ == 1) { + // Special case to avoid the nested for loop below. + (*this)[0].CopyTo(length, start_index, destination); + return length; + } + for (size_t i = 0; i < length; ++i) { + for (size_t channel = 0; channel < num_channels_; ++channel) { + destination[index] = (*this)[channel][i + start_index]; + ++index; + } + } + return index; +} + +size_t AudioMultiVector::ReadInterleavedFromEnd(size_t length, + int16_t* destination) const { + length = std::min(length, Size()); // Cannot read more than Size() elements. + return ReadInterleavedFromIndex(Size() - length, length, destination); +} + +void AudioMultiVector::OverwriteAt(const AudioMultiVector& insert_this, + size_t length, + size_t position) { + RTC_DCHECK_EQ(num_channels_, insert_this.num_channels_); + // Cap `length` at the length of `insert_this`. + RTC_DCHECK_LE(length, insert_this.Size()); + length = std::min(length, insert_this.Size()); + if (num_channels_ == insert_this.num_channels_) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->OverwriteAt(insert_this[i], length, position); + } + } +} + +void AudioMultiVector::CrossFade(const AudioMultiVector& append_this, + size_t fade_length) { + RTC_DCHECK_EQ(num_channels_, append_this.num_channels_); + if (num_channels_ == append_this.num_channels_) { + for (size_t i = 0; i < num_channels_; ++i) { + channels_[i]->CrossFade(append_this[i], fade_length); + } + } +} + +size_t AudioMultiVector::Channels() const { + return num_channels_; +} + +size_t AudioMultiVector::Size() const { + RTC_DCHECK(channels_[0]); + return channels_[0]->Size(); +} + +void AudioMultiVector::AssertSize(size_t required_size) { + if (Size() < required_size) { + size_t extend_length = required_size - Size(); + for (size_t channel = 0; channel < num_channels_; ++channel) { + channels_[channel]->Extend(extend_length); + } + } +} + +bool AudioMultiVector::Empty() const { + RTC_DCHECK(channels_[0]); + return channels_[0]->Empty(); +} + +void AudioMultiVector::CopyChannel(size_t from_channel, size_t to_channel) { + RTC_DCHECK_LT(from_channel, num_channels_); + RTC_DCHECK_LT(to_channel, num_channels_); + channels_[from_channel]->CopyTo(channels_[to_channel]); +} + +const AudioVector& AudioMultiVector::operator[](size_t index) const { + return *(channels_[index]); +} + +AudioVector& AudioMultiVector::operator[](size_t index) { + return *(channels_[index]); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.h b/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.h new file mode 100644 index 0000000000..715ec6dfc7 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_AUDIO_MULTI_VECTOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_AUDIO_MULTI_VECTOR_H_ + +#include +#include + +#include + +#include "api/array_view.h" +#include "modules/audio_coding/neteq/audio_vector.h" + +namespace webrtc { + +class AudioMultiVector { + public: + // Creates an empty AudioMultiVector with `N` audio channels. `N` must be + // larger than 0. + explicit AudioMultiVector(size_t N); + + // Creates an AudioMultiVector with `N` audio channels, each channel having + // an initial size. `N` must be larger than 0. + AudioMultiVector(size_t N, size_t initial_size); + + virtual ~AudioMultiVector(); + + AudioMultiVector(const AudioMultiVector&) = delete; + AudioMultiVector& operator=(const AudioMultiVector&) = delete; + + // Deletes all values and make the vector empty. + virtual void Clear(); + + // Clears the vector and inserts `length` zeros into each channel. + virtual void Zeros(size_t length); + + // Copies all values from this vector to `copy_to`. Any contents in `copy_to` + // are deleted. After the operation is done, `copy_to` will be an exact + // replica of this object. The source and the destination must have the same + // number of channels. + virtual void CopyTo(AudioMultiVector* copy_to) const; + + // Appends the contents of `append_this` to the end of this object. The array + // is assumed to be channel-interleaved. The length must be an even multiple + // of this object's number of channels. The length of this object is increased + // with the length of the array divided by the number of channels. + void PushBackInterleaved(rtc::ArrayView append_this); + + // Appends the contents of AudioMultiVector `append_this` to this object. The + // length of this object is increased with the length of `append_this`. + virtual void PushBack(const AudioMultiVector& append_this); + + // Appends the contents of AudioMultiVector `append_this` to this object, + // taken from `index` up until the end of `append_this`. The length of this + // object is increased. + virtual void PushBackFromIndex(const AudioMultiVector& append_this, + size_t index); + + // Removes `length` elements from the beginning of this object, from each + // channel. + virtual void PopFront(size_t length); + + // Removes `length` elements from the end of this object, from each + // channel. + virtual void PopBack(size_t length); + + // Reads `length` samples from each channel and writes them interleaved to + // `destination`. The total number of elements written to `destination` is + // returned, i.e., `length` * number of channels. If the AudioMultiVector + // contains less than `length` samples per channel, this is reflected in the + // return value. + virtual size_t ReadInterleaved(size_t length, int16_t* destination) const; + + // Like ReadInterleaved() above, but reads from `start_index` instead of from + // the beginning. + virtual size_t ReadInterleavedFromIndex(size_t start_index, + size_t length, + int16_t* destination) const; + + // Like ReadInterleaved() above, but reads from the end instead of from + // the beginning. + virtual size_t ReadInterleavedFromEnd(size_t length, + int16_t* destination) const; + + // Overwrites each channel in this AudioMultiVector with values taken from + // `insert_this`. The values are taken from the beginning of `insert_this` and + // are inserted starting at `position`. `length` values are written into each + // channel. If `length` and `position` are selected such that the new data + // extends beyond the end of the current AudioVector, the vector is extended + // to accommodate the new data. `length` is limited to the length of + // `insert_this`. + virtual void OverwriteAt(const AudioMultiVector& insert_this, + size_t length, + size_t position); + + // Appends `append_this` to the end of the current vector. Lets the two + // vectors overlap by `fade_length` samples (per channel), and cross-fade + // linearly in this region. + virtual void CrossFade(const AudioMultiVector& append_this, + size_t fade_length); + + // Returns the number of channels. + virtual size_t Channels() const; + + // Returns the number of elements per channel in this AudioMultiVector. + virtual size_t Size() const; + + // Verify that each channel can hold at least `required_size` elements. If + // not, extend accordingly. + virtual void AssertSize(size_t required_size); + + virtual bool Empty() const; + + // Copies the data between two channels in the AudioMultiVector. The method + // does not add any new channel. Thus, `from_channel` and `to_channel` must + // both be valid channel numbers. + virtual void CopyChannel(size_t from_channel, size_t to_channel); + + // Accesses and modifies a channel (i.e., an AudioVector object) of this + // AudioMultiVector. + const AudioVector& operator[](size_t index) const; + AudioVector& operator[](size_t index); + + protected: + std::vector channels_; + size_t num_channels_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_AUDIO_MULTI_VECTOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector_unittest.cc new file mode 100644 index 0000000000..386c3d48a3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector_unittest.cc @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/audio_multi_vector.h" + +#include + +#include +#include + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" + +namespace webrtc { + +// This is a value-parameterized test. The test cases are instantiated with +// different values for the test parameter, which is used to determine the +// number of channels in the AudioMultiBuffer. Note that it is not possible +// to combine typed testing with value-parameterized testing, and since the +// tests for AudioVector already covers a number of different type parameters, +// this test focuses on testing different number of channels, and keeping the +// value type constant. + +class AudioMultiVectorTest : public ::testing::TestWithParam { + protected: + AudioMultiVectorTest() + : num_channels_(GetParam()), // Get the test parameter. + array_interleaved_(num_channels_ * array_length()) {} + + ~AudioMultiVectorTest() = default; + + virtual void SetUp() { + // Populate test arrays. + for (size_t i = 0; i < array_length(); ++i) { + array_[i] = static_cast(i); + } + int16_t* ptr = array_interleaved_.data(); + // Write 100, 101, 102, ... for first channel. + // Write 200, 201, 202, ... for second channel. + // And so on. + for (size_t i = 0; i < array_length(); ++i) { + for (size_t j = 1; j <= num_channels_; ++j) { + *ptr = rtc::checked_cast(j * 100 + i); + ++ptr; + } + } + } + + size_t array_length() const { return sizeof(array_) / sizeof(array_[0]); } + + const size_t num_channels_; + int16_t array_[10]; + std::vector array_interleaved_; +}; + +// Create and destroy AudioMultiVector objects, both empty and with a predefined +// length. +TEST_P(AudioMultiVectorTest, CreateAndDestroy) { + AudioMultiVector vec1(num_channels_); + EXPECT_TRUE(vec1.Empty()); + EXPECT_EQ(num_channels_, vec1.Channels()); + EXPECT_EQ(0u, vec1.Size()); + + size_t initial_size = 17; + AudioMultiVector vec2(num_channels_, initial_size); + EXPECT_FALSE(vec2.Empty()); + EXPECT_EQ(num_channels_, vec2.Channels()); + EXPECT_EQ(initial_size, vec2.Size()); +} + +// Test the subscript operator [] for getting and setting. +TEST_P(AudioMultiVectorTest, SubscriptOperator) { + AudioMultiVector vec(num_channels_, array_length()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < array_length(); ++i) { + vec[channel][i] = static_cast(i); + // Make sure to use the const version. + const AudioVector& audio_vec = vec[channel]; + EXPECT_EQ(static_cast(i), audio_vec[i]); + } + } +} + +// Test the PushBackInterleaved method and the CopyFrom method. The Clear +// method is also invoked. +TEST_P(AudioMultiVectorTest, PushBackInterleavedAndCopy) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_); + AudioMultiVector vec_copy(num_channels_); + vec.CopyTo(&vec_copy); // Copy from `vec` to `vec_copy`. + ASSERT_EQ(num_channels_, vec.Channels()); + ASSERT_EQ(array_length(), vec.Size()); + ASSERT_EQ(num_channels_, vec_copy.Channels()); + ASSERT_EQ(array_length(), vec_copy.Size()); + for (size_t channel = 0; channel < vec.Channels(); ++channel) { + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(static_cast((channel + 1) * 100 + i), vec[channel][i]); + EXPECT_EQ(vec[channel][i], vec_copy[channel][i]); + } + } + + // Clear `vec` and verify that it is empty. + vec.Clear(); + EXPECT_TRUE(vec.Empty()); + + // Now copy the empty vector and verify that the copy becomes empty too. + vec.CopyTo(&vec_copy); + EXPECT_TRUE(vec_copy.Empty()); +} + +// Try to copy to a NULL pointer. Nothing should happen. +TEST_P(AudioMultiVectorTest, CopyToNull) { + AudioMultiVector vec(num_channels_); + AudioMultiVector* vec_copy = NULL; + vec.PushBackInterleaved(array_interleaved_); + vec.CopyTo(vec_copy); +} + +// Test the PushBack method with another AudioMultiVector as input argument. +TEST_P(AudioMultiVectorTest, PushBackVector) { + AudioMultiVector vec1(num_channels_, array_length()); + AudioMultiVector vec2(num_channels_, array_length()); + // Set the first vector to [0, 1, ..., array_length() - 1] + + // 100 * channel_number. + // Set the second vector to [array_length(), array_length() + 1, ..., + // 2 * array_length() - 1] + 100 * channel_number. + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < array_length(); ++i) { + vec1[channel][i] = static_cast(i + 100 * channel); + vec2[channel][i] = + static_cast(i + 100 * channel + array_length()); + } + } + // Append vec2 to the back of vec1. + vec1.PushBack(vec2); + ASSERT_EQ(2u * array_length(), vec1.Size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < 2 * array_length(); ++i) { + EXPECT_EQ(static_cast(i + 100 * channel), vec1[channel][i]); + } + } +} + +// Test the PushBackFromIndex method. +TEST_P(AudioMultiVectorTest, PushBackFromIndex) { + AudioMultiVector vec1(num_channels_); + vec1.PushBackInterleaved(array_interleaved_); + AudioMultiVector vec2(num_channels_); + + // Append vec1 to the back of vec2 (which is empty). Read vec1 from the second + // last element. + vec2.PushBackFromIndex(vec1, array_length() - 2); + ASSERT_EQ(2u, vec2.Size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < 2; ++i) { + EXPECT_EQ(array_interleaved_[channel + + num_channels_ * (array_length() - 2 + i)], + vec2[channel][i]); + } + } +} + +// Starts with pushing some values to the vector, then test the Zeros method. +TEST_P(AudioMultiVectorTest, Zeros) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_); + vec.Zeros(2 * array_length()); + ASSERT_EQ(num_channels_, vec.Channels()); + ASSERT_EQ(2u * array_length(), vec.Size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + for (size_t i = 0; i < 2 * array_length(); ++i) { + EXPECT_EQ(0, vec[channel][i]); + } + } +} + +// Test the ReadInterleaved method +TEST_P(AudioMultiVectorTest, ReadInterleaved) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_); + int16_t* output = new int16_t[array_interleaved_.size()]; + // Read 5 samples. + size_t read_samples = 5; + EXPECT_EQ(num_channels_ * read_samples, + vec.ReadInterleaved(read_samples, output)); + EXPECT_EQ(0, memcmp(array_interleaved_.data(), output, + read_samples * sizeof(int16_t))); + + // Read too many samples. Expect to get all samples from the vector. + EXPECT_EQ(array_interleaved_.size(), + vec.ReadInterleaved(array_length() + 1, output)); + EXPECT_EQ(0, memcmp(array_interleaved_.data(), output, + read_samples * sizeof(int16_t))); + + delete[] output; +} + +// Test the PopFront method. +TEST_P(AudioMultiVectorTest, PopFront) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_); + vec.PopFront(1); // Remove one element from each channel. + ASSERT_EQ(array_length() - 1u, vec.Size()); + // Let `ptr` point to the second element of the first channel in the + // interleaved array. + int16_t* ptr = &array_interleaved_[num_channels_]; + for (size_t i = 0; i < array_length() - 1; ++i) { + for (size_t channel = 0; channel < num_channels_; ++channel) { + EXPECT_EQ(*ptr, vec[channel][i]); + ++ptr; + } + } + vec.PopFront(array_length()); // Remove more elements than vector size. + EXPECT_EQ(0u, vec.Size()); +} + +// Test the PopBack method. +TEST_P(AudioMultiVectorTest, PopBack) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_); + vec.PopBack(1); // Remove one element from each channel. + ASSERT_EQ(array_length() - 1u, vec.Size()); + // Let `ptr` point to the first element of the first channel in the + // interleaved array. + int16_t* ptr = array_interleaved_.data(); + for (size_t i = 0; i < array_length() - 1; ++i) { + for (size_t channel = 0; channel < num_channels_; ++channel) { + EXPECT_EQ(*ptr, vec[channel][i]); + ++ptr; + } + } + vec.PopBack(array_length()); // Remove more elements than vector size. + EXPECT_EQ(0u, vec.Size()); +} + +// Test the AssertSize method. +TEST_P(AudioMultiVectorTest, AssertSize) { + AudioMultiVector vec(num_channels_, array_length()); + EXPECT_EQ(array_length(), vec.Size()); + // Start with asserting with smaller sizes than already allocated. + vec.AssertSize(0); + vec.AssertSize(array_length() - 1); + // Nothing should have changed. + EXPECT_EQ(array_length(), vec.Size()); + // Assert with one element longer than already allocated. + vec.AssertSize(array_length() + 1); + // Expect vector to have grown. + EXPECT_EQ(array_length() + 1, vec.Size()); + // Also check the individual AudioVectors. + for (size_t channel = 0; channel < vec.Channels(); ++channel) { + EXPECT_EQ(array_length() + 1u, vec[channel].Size()); + } +} + +// Test the PushBack method with another AudioMultiVector as input argument. +TEST_P(AudioMultiVectorTest, OverwriteAt) { + AudioMultiVector vec1(num_channels_); + vec1.PushBackInterleaved(array_interleaved_); + AudioMultiVector vec2(num_channels_); + vec2.Zeros(3); // 3 zeros in each channel. + // Overwrite vec2 at position 5. + vec1.OverwriteAt(vec2, 3, 5); + // Verify result. + // Length remains the same. + ASSERT_EQ(array_length(), vec1.Size()); + int16_t* ptr = array_interleaved_.data(); + for (size_t i = 0; i < array_length() - 1; ++i) { + for (size_t channel = 0; channel < num_channels_; ++channel) { + if (i >= 5 && i <= 7) { + // Elements 5, 6, 7 should have been replaced with zeros. + EXPECT_EQ(0, vec1[channel][i]); + } else { + EXPECT_EQ(*ptr, vec1[channel][i]); + } + ++ptr; + } + } +} + +// Test the CopyChannel method, when the test is instantiated with at least two +// channels. +TEST_P(AudioMultiVectorTest, CopyChannel) { + if (num_channels_ < 2) + return; + + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved(array_interleaved_); + // Create a reference copy. + AudioMultiVector ref(num_channels_); + ref.PushBack(vec); + // Copy from first to last channel. + vec.CopyChannel(0, num_channels_ - 1); + // Verify that the first and last channels are identical; the others should + // be left untouched. + for (size_t i = 0; i < array_length(); ++i) { + // Verify that all but the last channel are untouched. + for (size_t channel = 0; channel < num_channels_ - 1; ++channel) { + EXPECT_EQ(ref[channel][i], vec[channel][i]); + } + // Verify that the last and the first channels are identical. + EXPECT_EQ(vec[0][i], vec[num_channels_ - 1][i]); + } +} + +TEST_P(AudioMultiVectorTest, PushBackEmptyArray) { + AudioMultiVector vec(num_channels_); + vec.PushBackInterleaved({}); + EXPECT_TRUE(vec.Empty()); +} + +INSTANTIATE_TEST_SUITE_P(TestNumChannels, + AudioMultiVectorTest, + ::testing::Values(static_cast(1), + static_cast(2), + static_cast(5))); +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.cc b/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.cc new file mode 100644 index 0000000000..10e8936447 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.cc @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/audio_vector.h" + + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +AudioVector::AudioVector() : AudioVector(kDefaultInitialSize) { + Clear(); +} + +AudioVector::AudioVector(size_t initial_size) + : array_(new int16_t[initial_size + 1]), + capacity_(initial_size + 1), + begin_index_(0), + end_index_(capacity_ - 1) { + memset(array_.get(), 0, capacity_ * sizeof(int16_t)); +} + +AudioVector::~AudioVector() = default; + +void AudioVector::Clear() { + end_index_ = begin_index_ = 0; +} + +void AudioVector::CopyTo(AudioVector* copy_to) const { + RTC_DCHECK(copy_to); + copy_to->Reserve(Size()); + CopyTo(Size(), 0, copy_to->array_.get()); + copy_to->begin_index_ = 0; + copy_to->end_index_ = Size(); +} + +void AudioVector::CopyTo(size_t length, + size_t position, + int16_t* copy_to) const { + if (length == 0) + return; + length = std::min(length, Size() - position); + const size_t copy_index = (begin_index_ + position) % capacity_; + const size_t first_chunk_length = std::min(length, capacity_ - copy_index); + memcpy(copy_to, &array_[copy_index], first_chunk_length * sizeof(int16_t)); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + memcpy(©_to[first_chunk_length], array_.get(), + remaining_length * sizeof(int16_t)); + } +} + +void AudioVector::PushFront(const AudioVector& prepend_this) { + const size_t length = prepend_this.Size(); + if (length == 0) + return; + + // Although the subsequent calling to PushFront does Reserve in it, it is + // always more efficient to do a big Reserve first. + Reserve(Size() + length); + + const size_t first_chunk_length = + std::min(length, prepend_this.capacity_ - prepend_this.begin_index_); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) + PushFront(prepend_this.array_.get(), remaining_length); + PushFront(&prepend_this.array_[prepend_this.begin_index_], + first_chunk_length); +} + +void AudioVector::PushFront(const int16_t* prepend_this, size_t length) { + if (length == 0) + return; + Reserve(Size() + length); + const size_t first_chunk_length = std::min(length, begin_index_); + memcpy(&array_[begin_index_ - first_chunk_length], + &prepend_this[length - first_chunk_length], + first_chunk_length * sizeof(int16_t)); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + memcpy(&array_[capacity_ - remaining_length], prepend_this, + remaining_length * sizeof(int16_t)); + } + begin_index_ = (begin_index_ + capacity_ - length) % capacity_; +} + +void AudioVector::PushBack(const AudioVector& append_this) { + PushBack(append_this, append_this.Size(), 0); +} + +void AudioVector::PushBack(const AudioVector& append_this, + size_t length, + size_t position) { + RTC_DCHECK_LE(position, append_this.Size()); + RTC_DCHECK_LE(length, append_this.Size() - position); + + if (length == 0) + return; + + // Although the subsequent calling to PushBack does Reserve in it, it is + // always more efficient to do a big Reserve first. + Reserve(Size() + length); + + const size_t start_index = + (append_this.begin_index_ + position) % append_this.capacity_; + const size_t first_chunk_length = + std::min(length, append_this.capacity_ - start_index); + PushBack(&append_this.array_[start_index], first_chunk_length); + + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) + PushBack(append_this.array_.get(), remaining_length); +} + +void AudioVector::PushBack(const int16_t* append_this, size_t length) { + if (length == 0) + return; + Reserve(Size() + length); + const size_t first_chunk_length = std::min(length, capacity_ - end_index_); + memcpy(&array_[end_index_], append_this, + first_chunk_length * sizeof(int16_t)); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + memcpy(array_.get(), &append_this[first_chunk_length], + remaining_length * sizeof(int16_t)); + } + end_index_ = (end_index_ + length) % capacity_; +} + +void AudioVector::PopFront(size_t length) { + if (length == 0) + return; + length = std::min(length, Size()); + begin_index_ = (begin_index_ + length) % capacity_; +} + +void AudioVector::PopBack(size_t length) { + if (length == 0) + return; + // Never remove more than what is in the array. + length = std::min(length, Size()); + end_index_ = (end_index_ + capacity_ - length) % capacity_; +} + +void AudioVector::Extend(size_t extra_length) { + if (extra_length == 0) + return; + InsertZerosByPushBack(extra_length, Size()); +} + +void AudioVector::InsertAt(const int16_t* insert_this, + size_t length, + size_t position) { + if (length == 0) + return; + // Cap the insert position at the current array length. + position = std::min(Size(), position); + + // When inserting to a position closer to the beginning, it is more efficient + // to insert by pushing front than to insert by pushing back, since less data + // will be moved, vice versa. + if (position <= Size() - position) { + InsertByPushFront(insert_this, length, position); + } else { + InsertByPushBack(insert_this, length, position); + } +} + +void AudioVector::InsertZerosAt(size_t length, size_t position) { + if (length == 0) + return; + // Cap the insert position at the current array length. + position = std::min(Size(), position); + + // When inserting to a position closer to the beginning, it is more efficient + // to insert by pushing front than to insert by pushing back, since less data + // will be moved, vice versa. + if (position <= Size() - position) { + InsertZerosByPushFront(length, position); + } else { + InsertZerosByPushBack(length, position); + } +} + +void AudioVector::OverwriteAt(const AudioVector& insert_this, + size_t length, + size_t position) { + RTC_DCHECK_LE(length, insert_this.Size()); + if (length == 0) + return; + + // Cap the insert position at the current array length. + position = std::min(Size(), position); + + // Although the subsequent calling to OverwriteAt does Reserve in it, it is + // always more efficient to do a big Reserve first. + size_t new_size = std::max(Size(), position + length); + Reserve(new_size); + + const size_t first_chunk_length = + std::min(length, insert_this.capacity_ - insert_this.begin_index_); + OverwriteAt(&insert_this.array_[insert_this.begin_index_], first_chunk_length, + position); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + OverwriteAt(insert_this.array_.get(), remaining_length, + position + first_chunk_length); + } +} + +void AudioVector::OverwriteAt(const int16_t* insert_this, + size_t length, + size_t position) { + if (length == 0) + return; + // Cap the insert position at the current array length. + position = std::min(Size(), position); + + size_t new_size = std::max(Size(), position + length); + Reserve(new_size); + + const size_t overwrite_index = (begin_index_ + position) % capacity_; + const size_t first_chunk_length = + std::min(length, capacity_ - overwrite_index); + memcpy(&array_[overwrite_index], insert_this, + first_chunk_length * sizeof(int16_t)); + const size_t remaining_length = length - first_chunk_length; + if (remaining_length > 0) { + memcpy(array_.get(), &insert_this[first_chunk_length], + remaining_length * sizeof(int16_t)); + } + + end_index_ = (begin_index_ + new_size) % capacity_; +} + +void AudioVector::CrossFade(const AudioVector& append_this, + size_t fade_length) { + // Fade length cannot be longer than the current vector or `append_this`. + RTC_DCHECK_LE(fade_length, Size()); + RTC_DCHECK_LE(fade_length, append_this.Size()); + fade_length = std::min(fade_length, Size()); + fade_length = std::min(fade_length, append_this.Size()); + size_t position = Size() - fade_length + begin_index_; + // Cross fade the overlapping regions. + // `alpha` is the mixing factor in Q14. + // TODO(hlundin): Consider skipping +1 in the denominator to produce a + // smoother cross-fade, in particular at the end of the fade. + int alpha_step = 16384 / (static_cast(fade_length) + 1); + int alpha = 16384; + for (size_t i = 0; i < fade_length; ++i) { + alpha -= alpha_step; + array_[(position + i) % capacity_] = + (alpha * array_[(position + i) % capacity_] + + (16384 - alpha) * append_this[i] + 8192) >> + 14; + } + RTC_DCHECK_GE(alpha, 0); // Verify that the slope was correct. + // Append what is left of `append_this`. + size_t samples_to_push_back = append_this.Size() - fade_length; + if (samples_to_push_back > 0) + PushBack(append_this, samples_to_push_back, fade_length); +} + +// Returns the number of elements in this AudioVector. +size_t AudioVector::Size() const { + return (end_index_ + capacity_ - begin_index_) % capacity_; +} + +// Returns true if this AudioVector is empty. +bool AudioVector::Empty() const { + return begin_index_ == end_index_; +} + +void AudioVector::Reserve(size_t n) { + if (capacity_ > n) + return; + const size_t length = Size(); + // Reserve one more sample to remove the ambiguity between empty vector and + // full vector. Therefore `begin_index_` == `end_index_` indicates empty + // vector, and `begin_index_` == (`end_index_` + 1) % capacity indicates + // full vector. + std::unique_ptr temp_array(new int16_t[n + 1]); + CopyTo(length, 0, temp_array.get()); + array_.swap(temp_array); + begin_index_ = 0; + end_index_ = length; + capacity_ = n + 1; +} + +void AudioVector::InsertByPushBack(const int16_t* insert_this, + size_t length, + size_t position) { + const size_t move_chunk_length = Size() - position; + std::unique_ptr temp_array(nullptr); + if (move_chunk_length > 0) { + // TODO(minyue): see if it is possible to avoid copying to a buffer. + temp_array.reset(new int16_t[move_chunk_length]); + CopyTo(move_chunk_length, position, temp_array.get()); + PopBack(move_chunk_length); + } + + Reserve(Size() + length + move_chunk_length); + PushBack(insert_this, length); + if (move_chunk_length > 0) + PushBack(temp_array.get(), move_chunk_length); +} + +void AudioVector::InsertByPushFront(const int16_t* insert_this, + size_t length, + size_t position) { + std::unique_ptr temp_array(nullptr); + if (position > 0) { + // TODO(minyue): see if it is possible to avoid copying to a buffer. + temp_array.reset(new int16_t[position]); + CopyTo(position, 0, temp_array.get()); + PopFront(position); + } + + Reserve(Size() + length + position); + PushFront(insert_this, length); + if (position > 0) + PushFront(temp_array.get(), position); +} + +void AudioVector::InsertZerosByPushBack(size_t length, size_t position) { + const size_t move_chunk_length = Size() - position; + std::unique_ptr temp_array(nullptr); + if (move_chunk_length > 0) { + temp_array.reset(new int16_t[move_chunk_length]); + CopyTo(move_chunk_length, position, temp_array.get()); + PopBack(move_chunk_length); + } + + Reserve(Size() + length + move_chunk_length); + + const size_t first_zero_chunk_length = + std::min(length, capacity_ - end_index_); + memset(&array_[end_index_], 0, first_zero_chunk_length * sizeof(int16_t)); + const size_t remaining_zero_length = length - first_zero_chunk_length; + if (remaining_zero_length > 0) + memset(array_.get(), 0, remaining_zero_length * sizeof(int16_t)); + end_index_ = (end_index_ + length) % capacity_; + + if (move_chunk_length > 0) + PushBack(temp_array.get(), move_chunk_length); +} + +void AudioVector::InsertZerosByPushFront(size_t length, size_t position) { + std::unique_ptr temp_array(nullptr); + if (position > 0) { + temp_array.reset(new int16_t[position]); + CopyTo(position, 0, temp_array.get()); + PopFront(position); + } + + Reserve(Size() + length + position); + + const size_t first_zero_chunk_length = std::min(length, begin_index_); + memset(&array_[begin_index_ - first_zero_chunk_length], 0, + first_zero_chunk_length * sizeof(int16_t)); + const size_t remaining_zero_length = length - first_zero_chunk_length; + if (remaining_zero_length > 0) + memset(&array_[capacity_ - remaining_zero_length], 0, + remaining_zero_length * sizeof(int16_t)); + begin_index_ = (begin_index_ + capacity_ - length) % capacity_; + + if (position > 0) + PushFront(temp_array.get(), position); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.h b/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.h new file mode 100644 index 0000000000..d68f3ec6be --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_AUDIO_VECTOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_AUDIO_VECTOR_H_ + +#include + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +class AudioVector { + public: + // Creates an empty AudioVector. + AudioVector(); + + // Creates an AudioVector with an initial size. + explicit AudioVector(size_t initial_size); + + virtual ~AudioVector(); + + AudioVector(const AudioVector&) = delete; + AudioVector& operator=(const AudioVector&) = delete; + + // Deletes all values and make the vector empty. + virtual void Clear(); + + // Copies all values from this vector to `copy_to`. Any contents in `copy_to` + // are deleted before the copy operation. After the operation is done, + // `copy_to` will be an exact replica of this object. + virtual void CopyTo(AudioVector* copy_to) const; + + // Copies `length` values from `position` in this vector to `copy_to`. + virtual void CopyTo(size_t length, size_t position, int16_t* copy_to) const; + + // Prepends the contents of AudioVector `prepend_this` to this object. The + // length of this object is increased with the length of `prepend_this`. + virtual void PushFront(const AudioVector& prepend_this); + + // Same as above, but with an array `prepend_this` with `length` elements as + // source. + virtual void PushFront(const int16_t* prepend_this, size_t length); + + // Same as PushFront but will append to the end of this object. + virtual void PushBack(const AudioVector& append_this); + + // Appends a segment of `append_this` to the end of this object. The segment + // starts from `position` and has `length` samples. + virtual void PushBack(const AudioVector& append_this, + size_t length, + size_t position); + + // Same as PushFront but will append to the end of this object. + virtual void PushBack(const int16_t* append_this, size_t length); + + // Removes `length` elements from the beginning of this object. + virtual void PopFront(size_t length); + + // Removes `length` elements from the end of this object. + virtual void PopBack(size_t length); + + // Extends this object with `extra_length` elements at the end. The new + // elements are initialized to zero. + virtual void Extend(size_t extra_length); + + // Inserts `length` elements taken from the array `insert_this` and insert + // them at `position`. The length of the AudioVector is increased by `length`. + // `position` = 0 means that the new values are prepended to the vector. + // `position` = Size() means that the new values are appended to the vector. + virtual void InsertAt(const int16_t* insert_this, + size_t length, + size_t position); + + // Like InsertAt, but inserts `length` zero elements at `position`. + virtual void InsertZerosAt(size_t length, size_t position); + + // Overwrites `length` elements of this AudioVector starting from `position` + // with first values in `AudioVector`. The definition of `position` + // is the same as for InsertAt(). If `length` and `position` are selected + // such that the new data extends beyond the end of the current AudioVector, + // the vector is extended to accommodate the new data. + virtual void OverwriteAt(const AudioVector& insert_this, + size_t length, + size_t position); + + // Overwrites `length` elements of this AudioVector with values taken from the + // array `insert_this`, starting at `position`. The definition of `position` + // is the same as for InsertAt(). If `length` and `position` are selected + // such that the new data extends beyond the end of the current AudioVector, + // the vector is extended to accommodate the new data. + virtual void OverwriteAt(const int16_t* insert_this, + size_t length, + size_t position); + + // Appends `append_this` to the end of the current vector. Lets the two + // vectors overlap by `fade_length` samples, and cross-fade linearly in this + // region. + virtual void CrossFade(const AudioVector& append_this, size_t fade_length); + + // Returns the number of elements in this AudioVector. + virtual size_t Size() const; + + // Returns true if this AudioVector is empty. + virtual bool Empty() const; + + // Accesses and modifies an element of AudioVector. + inline const int16_t& operator[](size_t index) const { + return array_[WrapIndex(index, begin_index_, capacity_)]; + } + + inline int16_t& operator[](size_t index) { + return array_[WrapIndex(index, begin_index_, capacity_)]; + } + + private: + static const size_t kDefaultInitialSize = 10; + + // This method is used by the [] operators to calculate an index within the + // capacity of the array, but without using the modulo operation (%). + static inline size_t WrapIndex(size_t index, + size_t begin_index, + size_t capacity) { + RTC_DCHECK_LT(index, capacity); + RTC_DCHECK_LT(begin_index, capacity); + size_t ix = begin_index + index; + RTC_DCHECK_GE(ix, index); // Check for overflow. + if (ix >= capacity) { + ix -= capacity; + } + RTC_DCHECK_LT(ix, capacity); + return ix; + } + + void Reserve(size_t n); + + void InsertByPushBack(const int16_t* insert_this, + size_t length, + size_t position); + + void InsertByPushFront(const int16_t* insert_this, + size_t length, + size_t position); + + void InsertZerosByPushBack(size_t length, size_t position); + + void InsertZerosByPushFront(size_t length, size_t position); + + std::unique_ptr array_; + + size_t capacity_; // Allocated number of samples in the array. + + // The index of the first sample in `array_`, except when + // |begin_index_ == end_index_|, which indicates an empty buffer. + size_t begin_index_; + + // The index of the sample after the last sample in `array_`. + size_t end_index_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_AUDIO_VECTOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector_unittest.cc new file mode 100644 index 0000000000..ae9dd88606 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector_unittest.cc @@ -0,0 +1,384 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/audio_vector.h" + +#include + +#include + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" + +namespace webrtc { + +class AudioVectorTest : public ::testing::Test { + protected: + virtual void SetUp() { + // Populate test array. + for (size_t i = 0; i < array_length(); ++i) { + array_[i] = rtc::checked_cast(i); + } + } + + size_t array_length() const { return sizeof(array_) / sizeof(array_[0]); } + + int16_t array_[10]; +}; + +// Create and destroy AudioVector objects, both empty and with a predefined +// length. +TEST_F(AudioVectorTest, CreateAndDestroy) { + AudioVector vec1; + EXPECT_TRUE(vec1.Empty()); + EXPECT_EQ(0u, vec1.Size()); + + size_t initial_size = 17; + AudioVector vec2(initial_size); + EXPECT_FALSE(vec2.Empty()); + EXPECT_EQ(initial_size, vec2.Size()); +} + +// Test the subscript operator [] for getting and setting. +TEST_F(AudioVectorTest, SubscriptOperator) { + AudioVector vec(array_length()); + for (size_t i = 0; i < array_length(); ++i) { + vec[i] = static_cast(i); + const int16_t& value = vec[i]; // Make sure to use the const version. + EXPECT_EQ(static_cast(i), value); + } +} + +// Test the PushBack method and the CopyFrom method. The Clear method is also +// invoked. +TEST_F(AudioVectorTest, PushBackAndCopy) { + AudioVector vec; + AudioVector vec_copy; + vec.PushBack(array_, array_length()); + vec.CopyTo(&vec_copy); // Copy from `vec` to `vec_copy`. + ASSERT_EQ(array_length(), vec.Size()); + ASSERT_EQ(array_length(), vec_copy.Size()); + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[i]); + EXPECT_EQ(array_[i], vec_copy[i]); + } + + // Clear `vec` and verify that it is empty. + vec.Clear(); + EXPECT_TRUE(vec.Empty()); + + // Now copy the empty vector and verify that the copy becomes empty too. + vec.CopyTo(&vec_copy); + EXPECT_TRUE(vec_copy.Empty()); +} + +// Test the PushBack method with another AudioVector as input argument. +TEST_F(AudioVectorTest, PushBackVector) { + static const size_t kLength = 10; + AudioVector vec1(kLength); + AudioVector vec2(kLength); + // Set the first vector to [0, 1, ..., kLength - 1]. + // Set the second vector to [kLength, kLength + 1, ..., 2 * kLength - 1]. + for (size_t i = 0; i < kLength; ++i) { + vec1[i] = static_cast(i); + vec2[i] = static_cast(i + kLength); + } + // Append vec2 to the back of vec1. + vec1.PushBack(vec2); + ASSERT_EQ(2 * kLength, vec1.Size()); + for (size_t i = 0; i < 2 * kLength; ++i) { + EXPECT_EQ(static_cast(i), vec1[i]); + } +} + +// Test the PushFront method. +TEST_F(AudioVectorTest, PushFront) { + AudioVector vec; + vec.PushFront(array_, array_length()); + ASSERT_EQ(array_length(), vec.Size()); + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[i]); + } +} + +// Test the PushFront method with another AudioVector as input argument. +TEST_F(AudioVectorTest, PushFrontVector) { + static const size_t kLength = 10; + AudioVector vec1(kLength); + AudioVector vec2(kLength); + // Set the first vector to [0, 1, ..., kLength - 1]. + // Set the second vector to [kLength, kLength + 1, ..., 2 * kLength - 1]. + for (size_t i = 0; i < kLength; ++i) { + vec1[i] = static_cast(i); + vec2[i] = static_cast(i + kLength); + } + // Prepend vec1 to the front of vec2. + vec2.PushFront(vec1); + ASSERT_EQ(2 * kLength, vec2.Size()); + for (size_t i = 0; i < 2 * kLength; ++i) { + EXPECT_EQ(static_cast(i), vec2[i]); + } +} + +// Test the PopFront method. +TEST_F(AudioVectorTest, PopFront) { + AudioVector vec; + vec.PushBack(array_, array_length()); + vec.PopFront(1); // Remove one element. + EXPECT_EQ(array_length() - 1u, vec.Size()); + for (size_t i = 0; i < array_length() - 1; ++i) { + EXPECT_EQ(static_cast(i + 1), vec[i]); + } + vec.PopFront(array_length()); // Remove more elements than vector size. + EXPECT_EQ(0u, vec.Size()); +} + +// Test the PopBack method. +TEST_F(AudioVectorTest, PopBack) { + AudioVector vec; + vec.PushBack(array_, array_length()); + vec.PopBack(1); // Remove one element. + EXPECT_EQ(array_length() - 1u, vec.Size()); + for (size_t i = 0; i < array_length() - 1; ++i) { + EXPECT_EQ(static_cast(i), vec[i]); + } + vec.PopBack(array_length()); // Remove more elements than vector size. + EXPECT_EQ(0u, vec.Size()); +} + +// Test the Extend method. +TEST_F(AudioVectorTest, Extend) { + AudioVector vec; + vec.PushBack(array_, array_length()); + vec.Extend(5); // Extend with 5 elements, which should all be zeros. + ASSERT_EQ(array_length() + 5u, vec.Size()); + // Verify that all are zero. + for (size_t i = array_length(); i < array_length() + 5; ++i) { + EXPECT_EQ(0, vec[i]); + } +} + +// Test the InsertAt method with an insert position in the middle of the vector. +TEST_F(AudioVectorTest, InsertAt) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = 5; + vec.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {0, 1, ..., `insert_position` - 1, 100, 101, ..., 100 + kNewLength - 1, + // `insert_position`, `insert_position` + 1, ..., kLength - 1}. + size_t pos = 0; + for (int i = 0; i < insert_position; ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } + for (size_t i = insert_position; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } +} + +// Test the InsertZerosAt method with an insert position in the middle of the +// vector. Use the InsertAt method as reference. +TEST_F(AudioVectorTest, InsertZerosAt) { + AudioVector vec; + AudioVector vec_ref; + vec.PushBack(array_, array_length()); + vec_ref.PushBack(array_, array_length()); + static const int kNewLength = 5; + int insert_position = 5; + vec.InsertZerosAt(kNewLength, insert_position); + int16_t new_array[kNewLength] = {0}; // All zero elements. + vec_ref.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vectors are identical. + ASSERT_EQ(vec_ref.Size(), vec.Size()); + for (size_t i = 0; i < vec.Size(); ++i) { + EXPECT_EQ(vec_ref[i], vec[i]); + } +} + +// Test the InsertAt method with an insert position at the start of the vector. +TEST_F(AudioVectorTest, InsertAtBeginning) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = 0; + vec.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {100, 101, ..., 100 + kNewLength - 1, + // 0, 1, ..., kLength - 1}. + size_t pos = 0; + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } + for (size_t i = insert_position; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } +} + +// Test the InsertAt method with an insert position at the end of the vector. +TEST_F(AudioVectorTest, InsertAtEnd) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = rtc::checked_cast(array_length()); + vec.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {0, 1, ..., kLength - 1, 100, 101, ..., 100 + kNewLength - 1 }. + size_t pos = 0; + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } +} + +// Test the InsertAt method with an insert position beyond the end of the +// vector. Verify that a position beyond the end of the vector does not lead to +// an error. The expected outcome is the same as if the vector end was used as +// input position. That is, the input position should be capped at the maximum +// allowed value. +TEST_F(AudioVectorTest, InsertBeyondEnd) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = + rtc::checked_cast(array_length() + 10); // Too large. + vec.InsertAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {0, 1, ..., kLength - 1, 100, 101, ..., 100 + kNewLength - 1 }. + size_t pos = 0; + for (size_t i = 0; i < array_length(); ++i) { + EXPECT_EQ(array_[i], vec[pos]); + ++pos; + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } +} + +// Test the OverwriteAt method with a position such that all of the new values +// fit within the old vector. +TEST_F(AudioVectorTest, OverwriteAt) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + size_t insert_position = 2; + vec.OverwriteAt(new_array, kNewLength, insert_position); + // Verify that the vector looks as follows: + // {0, ..., `insert_position` - 1, 100, 101, ..., 100 + kNewLength - 1, + // `insert_position`, `insert_position` + 1, ..., kLength - 1}. + size_t pos = 0; + for (pos = 0; pos < insert_position; ++pos) { + EXPECT_EQ(array_[pos], vec[pos]); + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } + for (; pos < array_length(); ++pos) { + EXPECT_EQ(array_[pos], vec[pos]); + } +} + +// Test the OverwriteAt method with a position such that some of the new values +// extend beyond the end of the current vector. This is valid, and the vector is +// expected to expand to accommodate the new values. +TEST_F(AudioVectorTest, OverwriteBeyondEnd) { + AudioVector vec; + vec.PushBack(array_, array_length()); + static const int kNewLength = 5; + int16_t new_array[kNewLength]; + // Set array elements to {100, 101, 102, ... }. + for (int i = 0; i < kNewLength; ++i) { + new_array[i] = 100 + i; + } + int insert_position = rtc::checked_cast(array_length() - 2); + vec.OverwriteAt(new_array, kNewLength, insert_position); + ASSERT_EQ(array_length() - 2u + kNewLength, vec.Size()); + // Verify that the vector looks as follows: + // {0, ..., `insert_position` - 1, 100, 101, ..., 100 + kNewLength - 1, + // `insert_position`, `insert_position` + 1, ..., kLength - 1}. + int pos = 0; + for (pos = 0; pos < insert_position; ++pos) { + EXPECT_EQ(array_[pos], vec[pos]); + } + for (int i = 0; i < kNewLength; ++i) { + EXPECT_EQ(new_array[i], vec[pos]); + ++pos; + } + // Verify that we checked to the end of `vec`. + EXPECT_EQ(vec.Size(), static_cast(pos)); +} + +TEST_F(AudioVectorTest, CrossFade) { + static const size_t kLength = 100; + static const size_t kFadeLength = 10; + AudioVector vec1(kLength); + AudioVector vec2(kLength); + // Set all vector elements to 0 in `vec1` and 100 in `vec2`. + for (size_t i = 0; i < kLength; ++i) { + vec1[i] = 0; + vec2[i] = 100; + } + vec1.CrossFade(vec2, kFadeLength); + ASSERT_EQ(2 * kLength - kFadeLength, vec1.Size()); + // First part untouched. + for (size_t i = 0; i < kLength - kFadeLength; ++i) { + EXPECT_EQ(0, vec1[i]); + } + // Check mixing zone. + for (size_t i = 0; i < kFadeLength; ++i) { + EXPECT_NEAR((i + 1) * 100 / (kFadeLength + 1), + vec1[kLength - kFadeLength + i], 1); + } + // Second part untouched. + for (size_t i = kLength; i < vec1.Size(); ++i) { + EXPECT_EQ(100, vec1[i]); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/background_noise.cc b/third_party/libwebrtc/modules/audio_coding/neteq/background_noise.cc new file mode 100644 index 0000000000..2c95d3b390 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/background_noise.cc @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/background_noise.h" + +#include // memcpy + +#include // min, max + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/cross_correlation.h" +#include "modules/audio_coding/neteq/post_decode_vad.h" + +namespace webrtc { +namespace { + +constexpr size_t kMaxSampleRate = 48000; + +} // namespace + +// static +constexpr size_t BackgroundNoise::kMaxLpcOrder; + +BackgroundNoise::BackgroundNoise(size_t num_channels) + : num_channels_(num_channels), + channel_parameters_(new ChannelParameters[num_channels_]) { + Reset(); +} + +BackgroundNoise::~BackgroundNoise() {} + +void BackgroundNoise::Reset() { + initialized_ = false; + for (size_t channel = 0; channel < num_channels_; ++channel) { + channel_parameters_[channel].Reset(); + } +} + +bool BackgroundNoise::Update(const AudioMultiVector& input, + const PostDecodeVad& vad) { + bool filter_params_saved = false; + if (vad.running() && vad.active_speech()) { + // Do not update the background noise parameters if we know that the signal + // is active speech. + return filter_params_saved; + } + + int32_t auto_correlation[kMaxLpcOrder + 1]; + int16_t fiter_output[kMaxLpcOrder + kResidualLength]; + int16_t reflection_coefficients[kMaxLpcOrder]; + int16_t lpc_coefficients[kMaxLpcOrder + 1]; + + for (size_t channel_ix = 0; channel_ix < num_channels_; ++channel_ix) { + ChannelParameters& parameters = channel_parameters_[channel_ix]; + int16_t temp_signal_array[kVecLen + kMaxLpcOrder] = {0}; + int16_t* temp_signal = &temp_signal_array[kMaxLpcOrder]; + RTC_DCHECK_GE(input.Size(), kVecLen); + input[channel_ix].CopyTo(kVecLen, input.Size() - kVecLen, temp_signal); + int32_t sample_energy = + CalculateAutoCorrelation(temp_signal, kVecLen, auto_correlation); + + if ((!vad.running() && + sample_energy < parameters.energy_update_threshold) || + (vad.running() && !vad.active_speech())) { + // Generate LPC coefficients. + if (auto_correlation[0] <= 0) { + // Center value in auto-correlation is not positive. Do not update. + return filter_params_saved; + } + + // Regardless of whether the filter is actually updated or not, + // update energy threshold levels, since we have in fact observed + // a low energy signal. + if (sample_energy < parameters.energy_update_threshold) { + // Never go under 1.0 in average sample energy. + parameters.energy_update_threshold = std::max(sample_energy, 1); + parameters.low_energy_update_threshold = 0; + } + + // Only update BGN if filter is stable, i.e., if return value from + // Levinson-Durbin function is 1. + if (WebRtcSpl_LevinsonDurbin(auto_correlation, lpc_coefficients, + reflection_coefficients, + kMaxLpcOrder) != 1) { + return filter_params_saved; + } + + // Generate the CNG gain factor by looking at the energy of the residual. + WebRtcSpl_FilterMAFastQ12(temp_signal + kVecLen - kResidualLength, + fiter_output, lpc_coefficients, + kMaxLpcOrder + 1, kResidualLength); + int32_t residual_energy = WebRtcSpl_DotProductWithScale( + fiter_output, fiter_output, kResidualLength, 0); + + // Check spectral flatness. + // Comparing the residual variance with the input signal variance tells + // if the spectrum is flat or not. + // If 5 * residual_energy >= 16 * sample_energy, the spectrum is flat + // enough. Also ensure that the energy is non-zero. + if ((sample_energy > 0) && + (int64_t{5} * residual_energy >= int64_t{16} * sample_energy)) { + // Spectrum is flat enough; save filter parameters. + // `temp_signal` + `kVecLen` - `kMaxLpcOrder` points at the first of the + // `kMaxLpcOrder` samples in the residual signal, which will form the + // filter state for the next noise generation. + SaveParameters(channel_ix, lpc_coefficients, + temp_signal + kVecLen - kMaxLpcOrder, sample_energy, + residual_energy); + filter_params_saved = true; + } + } else { + // Will only happen if post-decode VAD is disabled and `sample_energy` is + // not low enough. Increase the threshold for update so that it increases + // by a factor 4 in 4 seconds. + IncrementEnergyThreshold(channel_ix, sample_energy); + } + } + return filter_params_saved; +} + +void BackgroundNoise::GenerateBackgroundNoise( + rtc::ArrayView random_vector, + size_t channel, + int mute_slope, + bool too_many_expands, + size_t num_noise_samples, + int16_t* buffer) { + constexpr size_t kNoiseLpcOrder = kMaxLpcOrder; + int16_t scaled_random_vector[kMaxSampleRate / 8000 * 125]; + RTC_DCHECK_LE(num_noise_samples, (kMaxSampleRate / 8000 * 125)); + RTC_DCHECK_GE(random_vector.size(), num_noise_samples); + int16_t* noise_samples = &buffer[kNoiseLpcOrder]; + if (initialized()) { + // Use background noise parameters. + memcpy(noise_samples - kNoiseLpcOrder, FilterState(channel), + sizeof(int16_t) * kNoiseLpcOrder); + + int dc_offset = 0; + if (ScaleShift(channel) > 1) { + dc_offset = 1 << (ScaleShift(channel) - 1); + } + + // Scale random vector to correct energy level. + WebRtcSpl_AffineTransformVector(scaled_random_vector, random_vector.data(), + Scale(channel), dc_offset, + ScaleShift(channel), num_noise_samples); + + WebRtcSpl_FilterARFastQ12(scaled_random_vector, noise_samples, + Filter(channel), kNoiseLpcOrder + 1, + num_noise_samples); + + SetFilterState( + channel, + {&(noise_samples[num_noise_samples - kNoiseLpcOrder]), kNoiseLpcOrder}); + + // Unmute the background noise. + int16_t bgn_mute_factor = MuteFactor(channel); + if (bgn_mute_factor < 16384) { + WebRtcSpl_AffineTransformVector(noise_samples, noise_samples, + bgn_mute_factor, 8192, 14, + num_noise_samples); + } + // Update mute_factor in BackgroundNoise class. + SetMuteFactor(channel, bgn_mute_factor); + } else { + // BGN parameters have not been initialized; use zero noise. + memset(noise_samples, 0, sizeof(int16_t) * num_noise_samples); + } +} + +int32_t BackgroundNoise::Energy(size_t channel) const { + RTC_DCHECK_LT(channel, num_channels_); + return channel_parameters_[channel].energy; +} + +void BackgroundNoise::SetMuteFactor(size_t channel, int16_t value) { + RTC_DCHECK_LT(channel, num_channels_); + channel_parameters_[channel].mute_factor = value; +} + +int16_t BackgroundNoise::MuteFactor(size_t channel) const { + RTC_DCHECK_LT(channel, num_channels_); + return channel_parameters_[channel].mute_factor; +} + +const int16_t* BackgroundNoise::Filter(size_t channel) const { + RTC_DCHECK_LT(channel, num_channels_); + return channel_parameters_[channel].filter; +} + +const int16_t* BackgroundNoise::FilterState(size_t channel) const { + RTC_DCHECK_LT(channel, num_channels_); + return channel_parameters_[channel].filter_state; +} + +void BackgroundNoise::SetFilterState(size_t channel, + rtc::ArrayView input) { + RTC_DCHECK_LT(channel, num_channels_); + size_t length = std::min(input.size(), kMaxLpcOrder); + memcpy(channel_parameters_[channel].filter_state, input.data(), + length * sizeof(int16_t)); +} + +int16_t BackgroundNoise::Scale(size_t channel) const { + RTC_DCHECK_LT(channel, num_channels_); + return channel_parameters_[channel].scale; +} +int16_t BackgroundNoise::ScaleShift(size_t channel) const { + RTC_DCHECK_LT(channel, num_channels_); + return channel_parameters_[channel].scale_shift; +} + +int32_t BackgroundNoise::CalculateAutoCorrelation( + const int16_t* signal, + size_t length, + int32_t* auto_correlation) const { + static const int kCorrelationStep = -1; + const int correlation_scale = + CrossCorrelationWithAutoShift(signal, signal, length, kMaxLpcOrder + 1, + kCorrelationStep, auto_correlation); + + // Number of shifts to normalize energy to energy/sample. + int energy_sample_shift = kLogVecLen - correlation_scale; + return auto_correlation[0] >> energy_sample_shift; +} + +void BackgroundNoise::IncrementEnergyThreshold(size_t channel, + int32_t sample_energy) { + // TODO(hlundin): Simplify the below threshold update. What this code + // does is simply "threshold += (increment * threshold) >> 16", but due + // to the limited-width operations, it is not exactly the same. The + // difference should be inaudible, but bit-exactness would not be + // maintained. + RTC_DCHECK_LT(channel, num_channels_); + ChannelParameters& parameters = channel_parameters_[channel]; + int32_t temp_energy = + (kThresholdIncrement * parameters.low_energy_update_threshold) >> 16; + temp_energy += + kThresholdIncrement * (parameters.energy_update_threshold & 0xFF); + temp_energy += + (kThresholdIncrement * ((parameters.energy_update_threshold >> 8) & 0xFF)) + << 8; + parameters.low_energy_update_threshold += temp_energy; + + parameters.energy_update_threshold += + kThresholdIncrement * (parameters.energy_update_threshold >> 16); + parameters.energy_update_threshold += + parameters.low_energy_update_threshold >> 16; + parameters.low_energy_update_threshold = + parameters.low_energy_update_threshold & 0x0FFFF; + + // Update maximum energy. + // Decrease by a factor 1/1024 each time. + parameters.max_energy = parameters.max_energy - (parameters.max_energy >> 10); + if (sample_energy > parameters.max_energy) { + parameters.max_energy = sample_energy; + } + + // Set `energy_update_threshold` to no less than 60 dB lower than + // `max_energy_`. Adding 524288 assures proper rounding. + int32_t energy_update_threshold = (parameters.max_energy + 524288) >> 20; + if (energy_update_threshold > parameters.energy_update_threshold) { + parameters.energy_update_threshold = energy_update_threshold; + } +} + +void BackgroundNoise::SaveParameters(size_t channel, + const int16_t* lpc_coefficients, + const int16_t* filter_state, + int32_t sample_energy, + int32_t residual_energy) { + RTC_DCHECK_LT(channel, num_channels_); + ChannelParameters& parameters = channel_parameters_[channel]; + memcpy(parameters.filter, lpc_coefficients, + (kMaxLpcOrder + 1) * sizeof(int16_t)); + memcpy(parameters.filter_state, filter_state, kMaxLpcOrder * sizeof(int16_t)); + // Save energy level and update energy threshold levels. + // Never get under 1.0 in average sample energy. + parameters.energy = std::max(sample_energy, 1); + parameters.energy_update_threshold = parameters.energy; + parameters.low_energy_update_threshold = 0; + + // Normalize residual_energy to 29 or 30 bits before sqrt. + int16_t norm_shift = WebRtcSpl_NormW32(residual_energy) - 1; + if (norm_shift & 0x1) { + norm_shift -= 1; // Even number of shifts required. + } + residual_energy = WEBRTC_SPL_SHIFT_W32(residual_energy, norm_shift); + + // Calculate scale and shift factor. + parameters.scale = static_cast(WebRtcSpl_SqrtFloor(residual_energy)); + // Add 13 to the `scale_shift_`, since the random numbers table is in + // Q13. + // TODO(hlundin): Move the "13" to where the `scale_shift_` is used? + parameters.scale_shift = + static_cast(13 + ((kLogResidualLength + norm_shift) / 2)); + + initialized_ = true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/background_noise.h b/third_party/libwebrtc/modules/audio_coding/neteq/background_noise.h new file mode 100644 index 0000000000..8e6d5890a0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/background_noise.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_BACKGROUND_NOISE_H_ +#define MODULES_AUDIO_CODING_NETEQ_BACKGROUND_NOISE_H_ + +#include // size_t + +#include + +#include "api/array_view.h" + +namespace webrtc { + +// Forward declarations. +class AudioMultiVector; +class PostDecodeVad; + +// This class handles estimation of background noise parameters. +class BackgroundNoise { + public: + // TODO(hlundin): For 48 kHz support, increase kMaxLpcOrder to 10. + // Will work anyway, but probably sound a little worse. + static constexpr size_t kMaxLpcOrder = 8; // 32000 / 8000 + 4. + + explicit BackgroundNoise(size_t num_channels); + virtual ~BackgroundNoise(); + + BackgroundNoise(const BackgroundNoise&) = delete; + BackgroundNoise& operator=(const BackgroundNoise&) = delete; + + void Reset(); + + // Updates the parameter estimates based on the signal currently in the + // `sync_buffer`, and on the latest decision in `vad` if it is running. + // Returns true if the filter parameters are updated. + bool Update(const AudioMultiVector& sync_buffer, const PostDecodeVad& vad); + + // Generates background noise given a random vector and writes the output to + // `buffer`. + void GenerateBackgroundNoise(rtc::ArrayView random_vector, + size_t channel, + int mute_slope, + bool too_many_expands, + size_t num_noise_samples, + int16_t* buffer); + + // Returns `energy_` for `channel`. + int32_t Energy(size_t channel) const; + + // Sets the value of `mute_factor_` for `channel` to `value`. + void SetMuteFactor(size_t channel, int16_t value); + + // Returns `mute_factor_` for `channel`. + int16_t MuteFactor(size_t channel) const; + + // Returns a pointer to `filter_` for `channel`. + const int16_t* Filter(size_t channel) const; + + // Returns a pointer to `filter_state_` for `channel`. + const int16_t* FilterState(size_t channel) const; + + // Copies `input` to the filter state. Will not copy more than `kMaxLpcOrder` + // elements. + void SetFilterState(size_t channel, rtc::ArrayView input); + + // Returns `scale_` for `channel`. + int16_t Scale(size_t channel) const; + + // Returns `scale_shift_` for `channel`. + int16_t ScaleShift(size_t channel) const; + + // Accessors. + bool initialized() const { return initialized_; } + + private: + static const int kThresholdIncrement = 229; // 0.0035 in Q16. + static const size_t kVecLen = 256; + static const int kLogVecLen = 8; // log2(kVecLen). + static const size_t kResidualLength = 64; + static const int16_t kLogResidualLength = 6; // log2(kResidualLength) + + struct ChannelParameters { + // Constructor. + ChannelParameters() { Reset(); } + + void Reset() { + energy = 2500; + max_energy = 0; + energy_update_threshold = 500000; + low_energy_update_threshold = 0; + memset(filter_state, 0, sizeof(filter_state)); + memset(filter, 0, sizeof(filter)); + filter[0] = 4096; + mute_factor = 0; + scale = 20000; + scale_shift = 24; + } + + int32_t energy; + int32_t max_energy; + int32_t energy_update_threshold; + int32_t low_energy_update_threshold; + int16_t filter_state[kMaxLpcOrder]; + int16_t filter[kMaxLpcOrder + 1]; + int16_t mute_factor; + int16_t scale; + int16_t scale_shift; + }; + + int32_t CalculateAutoCorrelation(const int16_t* signal, + size_t length, + int32_t* auto_correlation) const; + + // Increments the energy threshold by a factor 1 + `kThresholdIncrement`. + void IncrementEnergyThreshold(size_t channel, int32_t sample_energy); + + // Updates the filter parameters. + void SaveParameters(size_t channel, + const int16_t* lpc_coefficients, + const int16_t* filter_state, + int32_t sample_energy, + int32_t residual_energy); + + size_t num_channels_; + std::unique_ptr channel_parameters_; + bool initialized_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_BACKGROUND_NOISE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/background_noise_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/background_noise_unittest.cc new file mode 100644 index 0000000000..e32492f57e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/background_noise_unittest.cc @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for BackgroundNoise class. + +#include "modules/audio_coding/neteq/background_noise.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(BackgroundNoise, CreateAndDestroy) { + size_t channels = 1; + BackgroundNoise bgn(channels); +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.cc b/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.cc new file mode 100644 index 0000000000..2c42d0d13f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.cc @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/buffer_level_filter.h" + +#include + +#include + +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +BufferLevelFilter::BufferLevelFilter() { + Reset(); +} + +void BufferLevelFilter::Reset() { + filtered_current_level_ = 0; + level_factor_ = 253; +} + +void BufferLevelFilter::Update(size_t buffer_size_samples, + int time_stretched_samples) { + // Filter: + // `filtered_current_level_` = `level_factor_` * `filtered_current_level_` + + // (1 - `level_factor_`) * `buffer_size_samples` + // `level_factor_` and `filtered_current_level_` are in Q8. + // `buffer_size_samples` is in Q0. + const int64_t filtered_current_level = + (level_factor_ * int64_t{filtered_current_level_} >> 8) + + (256 - level_factor_) * rtc::dchecked_cast(buffer_size_samples); + + // Account for time-scale operations (accelerate and pre-emptive expand) and + // make sure that the filtered value remains non-negative. + filtered_current_level_ = rtc::saturated_cast(std::max( + 0, filtered_current_level - int64_t{time_stretched_samples} * (1 << 8))); +} + +void BufferLevelFilter::SetFilteredBufferLevel(int buffer_size_samples) { + filtered_current_level_ = + rtc::saturated_cast(int64_t{buffer_size_samples} * 256); +} + +void BufferLevelFilter::SetTargetBufferLevel(int target_buffer_level_ms) { + if (target_buffer_level_ms <= 20) { + level_factor_ = 251; + } else if (target_buffer_level_ms <= 60) { + level_factor_ = 252; + } else if (target_buffer_level_ms <= 140) { + level_factor_ = 253; + } else { + level_factor_ = 254; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.h b/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.h new file mode 100644 index 0000000000..ced36da9c2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_BUFFER_LEVEL_FILTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_BUFFER_LEVEL_FILTER_H_ + +#include +#include + +namespace webrtc { + +class BufferLevelFilter { + public: + BufferLevelFilter(); + virtual ~BufferLevelFilter() {} + + BufferLevelFilter(const BufferLevelFilter&) = delete; + BufferLevelFilter& operator=(const BufferLevelFilter&) = delete; + + virtual void Reset(); + + // Updates the filter. Current buffer size is `buffer_size_samples`. + // `time_stretched_samples` is subtracted from the filtered value (thus + // bypassing the filter operation). + virtual void Update(size_t buffer_size_samples, int time_stretched_samples); + + // Set the filtered buffer level to a particular value directly. This should + // only be used in case of large changes in buffer size, such as buffer + // flushes. + virtual void SetFilteredBufferLevel(int buffer_size_samples); + + // The target level is used to select the appropriate filter coefficient. + virtual void SetTargetBufferLevel(int target_buffer_level_ms); + + // Returns filtered current level in number of samples. + virtual int filtered_current_level() const { + // Round to nearest whole sample. + return (int64_t{filtered_current_level_} + (1 << 7)) >> 8; + } + + private: + int level_factor_; // Filter factor for the buffer level filter in Q8. + int filtered_current_level_; // Filtered current buffer level in Q8. +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_BUFFER_LEVEL_FILTER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter_unittest.cc new file mode 100644 index 0000000000..6773e96f58 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter_unittest.cc @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for BufferLevelFilter class. + +#include "modules/audio_coding/neteq/buffer_level_filter.h" + +#include // Access to pow function. + +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(BufferLevelFilter, CreateAndDestroy) { + BufferLevelFilter* filter = new BufferLevelFilter(); + EXPECT_EQ(0, filter->filtered_current_level()); + delete filter; +} + +TEST(BufferLevelFilter, ConvergenceTest) { + BufferLevelFilter filter; + for (int times = 10; times <= 50; times += 10) { + for (int value = 100; value <= 200; value += 10) { + filter.Reset(); + filter.SetTargetBufferLevel(20); // Makes filter coefficient 251/256. + rtc::StringBuilder ss; + ss << "times = " << times << ", value = " << value; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + for (int i = 0; i < times; ++i) { + filter.Update(value, 0 /* time_stretched_samples */); + } + // Expect the filtered value to be (theoretically) + // (1 - (251/256) ^ `times`) * `value`. + double expected_value_double = (1 - pow(251.0 / 256.0, times)) * value; + int expected_value = static_cast(expected_value_double); + + // The actual value may differ slightly from the expected value due to + // intermediate-stage rounding errors in the filter implementation. + // This is why we have to use EXPECT_NEAR with a tolerance of +/-1. + EXPECT_NEAR(expected_value, filter.filtered_current_level(), 1); + } + } +} + +// Verify that target buffer level impacts on the filter convergence. +TEST(BufferLevelFilter, FilterFactor) { + BufferLevelFilter filter; + // Update 10 times with value 100. + const int kTimes = 10; + const int kValue = 100; + + filter.SetTargetBufferLevel(60); // Makes filter coefficient 252/256. + for (int i = 0; i < kTimes; ++i) { + filter.Update(kValue, 0 /* time_stretched_samples */); + } + // Expect the filtered value to be + // (1 - (252/256) ^ `kTimes`) * `kValue`. + int expected_value = 15; + EXPECT_EQ(expected_value, filter.filtered_current_level()); + + filter.Reset(); + filter.SetTargetBufferLevel(140); // Makes filter coefficient 253/256. + for (int i = 0; i < kTimes; ++i) { + filter.Update(kValue, 0 /* time_stretched_samples */); + } + // Expect the filtered value to be + // (1 - (253/256) ^ `kTimes`) * `kValue`. + expected_value = 11; + EXPECT_EQ(expected_value, filter.filtered_current_level()); + + filter.Reset(); + filter.SetTargetBufferLevel(160); // Makes filter coefficient 254/256. + for (int i = 0; i < kTimes; ++i) { + filter.Update(kValue, 0 /* time_stretched_samples */); + } + // Expect the filtered value to be + // (1 - (254/256) ^ `kTimes`) * `kValue`. + expected_value = 8; + EXPECT_EQ(expected_value, filter.filtered_current_level()); +} + +TEST(BufferLevelFilter, TimeStretchedSamples) { + BufferLevelFilter filter; + filter.SetTargetBufferLevel(20); // Makes filter coefficient 251/256. + // Update 10 times with value 100. + const int kTimes = 10; + const int kValue = 100; + const int kTimeStretchedSamples = 3; + for (int i = 0; i < kTimes; ++i) { + filter.Update(kValue, 0); + } + // Expect the filtered value to be + // (1 - (251/256) ^ `kTimes`) * `kValue`. + const int kExpectedValue = 18; + EXPECT_EQ(kExpectedValue, filter.filtered_current_level()); + + // Update filter again, now with non-zero value for packet length. + // Set the current filtered value to be the input, in order to isolate the + // impact of `kTimeStretchedSamples`. + filter.Update(filter.filtered_current_level(), kTimeStretchedSamples); + EXPECT_EQ(kExpectedValue - kTimeStretchedSamples, + filter.filtered_current_level()); + // Try negative value and verify that we come back to the previous result. + filter.Update(filter.filtered_current_level(), -kTimeStretchedSamples); + EXPECT_EQ(kExpectedValue, filter.filtered_current_level()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.cc b/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.cc new file mode 100644 index 0000000000..a2ce888f45 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.cc @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/comfort_noise.h" + + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/audio_vector.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/dsp_helper.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "rtc_base/buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +void ComfortNoise::Reset() { + first_call_ = true; +} + +int ComfortNoise::UpdateParameters(const Packet& packet) { + // Get comfort noise decoder. + if (decoder_database_->SetActiveCngDecoder(packet.payload_type) != kOK) { + return kUnknownPayloadType; + } + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + RTC_DCHECK(cng_decoder); + cng_decoder->UpdateSid(packet.payload); + return kOK; +} + +int ComfortNoise::Generate(size_t requested_length, AudioMultiVector* output) { + // TODO(hlundin): Change to an enumerator and skip assert. + RTC_DCHECK(fs_hz_ == 8000 || fs_hz_ == 16000 || fs_hz_ == 32000 || + fs_hz_ == 48000); + // Not adapted for multi-channel yet. + if (output->Channels() != 1) { + RTC_LOG(LS_ERROR) << "No multi-channel support"; + return kMultiChannelNotSupported; + } + + size_t number_of_samples = requested_length; + bool new_period = false; + if (first_call_) { + // Generate noise and overlap slightly with old data. + number_of_samples = requested_length + overlap_length_; + new_period = true; + } + output->AssertSize(number_of_samples); + // Get the decoder from the database. + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + if (!cng_decoder) { + RTC_LOG(LS_ERROR) << "Unknwown payload type"; + return kUnknownPayloadType; + } + + std::unique_ptr temp(new int16_t[number_of_samples]); + if (!cng_decoder->Generate( + rtc::ArrayView(temp.get(), number_of_samples), new_period)) { + // Error returned. + output->Zeros(requested_length); + RTC_LOG(LS_ERROR) + << "ComfortNoiseDecoder::Genererate failed to generate comfort noise"; + return kInternalError; + } + (*output)[0].OverwriteAt(temp.get(), number_of_samples, 0); + + if (first_call_) { + // Set tapering window parameters. Values are in Q15. + int16_t muting_window; // Mixing factor for overlap data. + int16_t muting_window_increment; // Mixing factor increment (negative). + int16_t unmuting_window; // Mixing factor for comfort noise. + int16_t unmuting_window_increment; // Mixing factor increment. + if (fs_hz_ == 8000) { + muting_window = DspHelper::kMuteFactorStart8kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement8kHz; + unmuting_window = DspHelper::kUnmuteFactorStart8kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement8kHz; + } else if (fs_hz_ == 16000) { + muting_window = DspHelper::kMuteFactorStart16kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement16kHz; + unmuting_window = DspHelper::kUnmuteFactorStart16kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement16kHz; + } else if (fs_hz_ == 32000) { + muting_window = DspHelper::kMuteFactorStart32kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement32kHz; + unmuting_window = DspHelper::kUnmuteFactorStart32kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement32kHz; + } else { // fs_hz_ == 48000 + muting_window = DspHelper::kMuteFactorStart48kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement48kHz; + unmuting_window = DspHelper::kUnmuteFactorStart48kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement48kHz; + } + + // Do overlap-add between new vector and overlap. + size_t start_ix = sync_buffer_->Size() - overlap_length_; + for (size_t i = 0; i < overlap_length_; i++) { + /* overlapVec[i] = WinMute * overlapVec[i] + WinUnMute * outData[i] */ + // The expression (*output)[0][i] is the i-th element in the first + // channel. + (*sync_buffer_)[0][start_ix + i] = + (((*sync_buffer_)[0][start_ix + i] * muting_window) + + ((*output)[0][i] * unmuting_window) + 16384) >> + 15; + muting_window += muting_window_increment; + unmuting_window += unmuting_window_increment; + } + // Remove `overlap_length_` samples from the front of `output` since they + // were mixed into `sync_buffer_` above. + output->PopFront(overlap_length_); + } + first_call_ = false; + return kOK; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.h b/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.h new file mode 100644 index 0000000000..31fcee31d0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_COMFORT_NOISE_H_ +#define MODULES_AUDIO_CODING_NETEQ_COMFORT_NOISE_H_ + +#include + +namespace webrtc { + +// Forward declarations. +class AudioMultiVector; +class DecoderDatabase; +class SyncBuffer; +struct Packet; + +// This class acts as an interface to the CNG generator. +class ComfortNoise { + public: + enum ReturnCodes { + kOK = 0, + kUnknownPayloadType, + kInternalError, + kMultiChannelNotSupported + }; + + ComfortNoise(int fs_hz, + DecoderDatabase* decoder_database, + SyncBuffer* sync_buffer) + : fs_hz_(fs_hz), + first_call_(true), + overlap_length_(5 * fs_hz_ / 8000), + decoder_database_(decoder_database), + sync_buffer_(sync_buffer) {} + + ComfortNoise(const ComfortNoise&) = delete; + ComfortNoise& operator=(const ComfortNoise&) = delete; + + // Resets the state. Should be called before each new comfort noise period. + void Reset(); + + // Update the comfort noise generator with the parameters in `packet`. + int UpdateParameters(const Packet& packet); + + // Generates `requested_length` samples of comfort noise and writes to + // `output`. If this is the first in call after Reset (or first after creating + // the object), it will also mix in comfort noise at the end of the + // SyncBuffer object provided in the constructor. + int Generate(size_t requested_length, AudioMultiVector* output); + + // Returns the last error code that was produced by the comfort noise + // decoder. Returns 0 if no error has been encountered since the last reset. + int internal_error_code() { return internal_error_code_; } + + private: + int fs_hz_; + bool first_call_; + size_t overlap_length_; + DecoderDatabase* decoder_database_; + SyncBuffer* sync_buffer_; + int internal_error_code_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_COMFORT_NOISE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise_unittest.cc new file mode 100644 index 0000000000..b436800061 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise_unittest.cc @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for ComfortNoise class. + +#include "modules/audio_coding/neteq/comfort_noise.h" + +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(ComfortNoise, CreateAndDestroy) { + int fs = 8000; + MockDecoderDatabase db; + SyncBuffer sync_buffer(1, 1000); + ComfortNoise cn(fs, &db, &sync_buffer); + EXPECT_CALL(db, Die()); // Called when `db` goes out of scope. +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.cc b/third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.cc new file mode 100644 index 0000000000..37ed9374f0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/cross_correlation.h" + +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" + +namespace webrtc { + +// This function decides the overflow-protecting scaling and calls +// WebRtcSpl_CrossCorrelation. +int CrossCorrelationWithAutoShift(const int16_t* sequence_1, + const int16_t* sequence_2, + size_t sequence_1_length, + size_t cross_correlation_length, + int cross_correlation_step, + int32_t* cross_correlation) { + // Find the element that has the maximum absolute value of sequence_1 and 2. + // Note that these values may be negative. + const int16_t max_1 = + WebRtcSpl_MaxAbsElementW16(sequence_1, sequence_1_length); + const int sequence_2_shift = + cross_correlation_step * (static_cast(cross_correlation_length) - 1); + const int16_t* sequence_2_start = + sequence_2_shift >= 0 ? sequence_2 : sequence_2 + sequence_2_shift; + const size_t sequence_2_length = + sequence_1_length + std::abs(sequence_2_shift); + const int16_t max_2 = + WebRtcSpl_MaxAbsElementW16(sequence_2_start, sequence_2_length); + + // In order to avoid overflow when computing the sum we should scale the + // samples so that (in_vector_length * max_1 * max_2) will not overflow. + const int64_t max_value = + abs(max_1 * max_2) * static_cast(sequence_1_length); + const int32_t factor = max_value >> 31; + const int scaling = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); + + WebRtcSpl_CrossCorrelation(cross_correlation, sequence_1, sequence_2, + sequence_1_length, cross_correlation_length, + scaling, cross_correlation_step); + + return scaling; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.h b/third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.h new file mode 100644 index 0000000000..5082ce6a30 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_CROSS_CORRELATION_H_ +#define MODULES_AUDIO_CODING_NETEQ_CROSS_CORRELATION_H_ + +#include +#include + +namespace webrtc { + +// The function calculates the cross-correlation between two sequences +// `sequence_1` and `sequence_2`. `sequence_1` is taken as reference, with +// `sequence_1_length` as its length. `sequence_2` slides for the calculation of +// cross-correlation. The result will be saved in `cross_correlation`. +// `cross_correlation_length` correlation points are calculated. +// The corresponding lag starts from 0, and increases with a step of +// `cross_correlation_step`. The result is without normalization. To avoid +// overflow, the result will be right shifted. The amount of shifts will be +// returned. +// +// Input: +// - sequence_1 : First sequence (reference). +// - sequence_2 : Second sequence (sliding during calculation). +// - sequence_1_length : Length of `sequence_1`. +// - cross_correlation_length : Number of cross-correlations to calculate. +// - cross_correlation_step : Step in the lag for the cross-correlation. +// +// Output: +// - cross_correlation : The cross-correlation in Q(-right_shifts) +// +// Return: +// Number of right shifts in cross_correlation. + +int CrossCorrelationWithAutoShift(const int16_t* sequence_1, + const int16_t* sequence_2, + size_t sequence_1_length, + size_t cross_correlation_length, + int cross_correlation_step, + int32_t* cross_correlation); + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_CROSS_CORRELATION_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.cc b/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.cc new file mode 100644 index 0000000000..91b0252d2c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.cc @@ -0,0 +1,515 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/decision_logic.h" + +#include + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/neteq/neteq.h" +#include "api/neteq/neteq_controller.h" +#include "modules/audio_coding/neteq/packet_arrival_history.h" +#include "modules/audio_coding/neteq/packet_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/experiments/struct_parameters_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { + +constexpr int kPostponeDecodingLevel = 50; +constexpr int kTargetLevelWindowMs = 100; +constexpr int kMaxWaitForPacketTicks = 10; +// The granularity of delay adjustments (accelerate/preemptive expand) is 15ms, +// but round up since the clock has a granularity of 10ms. +constexpr int kDelayAdjustmentGranularityMs = 20; + +std::unique_ptr CreateDelayManager( + const NetEqController::Config& neteq_config) { + DelayManager::Config config; + config.max_packets_in_buffer = neteq_config.max_packets_in_buffer; + config.base_minimum_delay_ms = neteq_config.base_min_delay_ms; + config.Log(); + return std::make_unique(config, neteq_config.tick_timer); +} + +bool IsTimestretch(NetEq::Mode mode) { + return mode == NetEq::Mode::kAccelerateSuccess || + mode == NetEq::Mode::kAccelerateLowEnergy || + mode == NetEq::Mode::kPreemptiveExpandSuccess || + mode == NetEq::Mode::kPreemptiveExpandLowEnergy; +} + +bool IsCng(NetEq::Mode mode) { + return mode == NetEq::Mode::kRfc3389Cng || + mode == NetEq::Mode::kCodecInternalCng; +} + +bool IsExpand(NetEq::Mode mode) { + return mode == NetEq::Mode::kExpand || mode == NetEq::Mode::kCodecPlc; +} + +} // namespace + +DecisionLogic::Config::Config() { + StructParametersParser::Create( + "enable_stable_playout_delay", &enable_stable_playout_delay, // + "reinit_after_expands", &reinit_after_expands, // + "packet_history_size_ms", &packet_history_size_ms, // + "cng_timeout_ms", &cng_timeout_ms, // + "deceleration_target_level_offset_ms", + &deceleration_target_level_offset_ms) + ->Parse(webrtc::field_trial::FindFullName( + "WebRTC-Audio-NetEqDecisionLogicConfig")); + RTC_LOG(LS_INFO) << "NetEq decision logic config:" + << " enable_stable_playout_delay=" + << enable_stable_playout_delay + << " reinit_after_expands=" << reinit_after_expands + << " packet_history_size_ms=" << packet_history_size_ms + << " cng_timeout_ms=" << cng_timeout_ms.value_or(-1) + << " deceleration_target_level_offset_ms=" + << deceleration_target_level_offset_ms; +} + +DecisionLogic::DecisionLogic(NetEqController::Config config) + : DecisionLogic(config, + CreateDelayManager(config), + std::make_unique()) {} + +DecisionLogic::DecisionLogic( + NetEqController::Config config, + std::unique_ptr delay_manager, + std::unique_ptr buffer_level_filter) + : delay_manager_(std::move(delay_manager)), + buffer_level_filter_(std::move(buffer_level_filter)), + packet_arrival_history_(config_.packet_history_size_ms), + tick_timer_(config.tick_timer), + disallow_time_stretching_(!config.allow_time_stretching), + timescale_countdown_( + tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1)) {} + +DecisionLogic::~DecisionLogic() = default; + +void DecisionLogic::SoftReset() { + packet_length_samples_ = 0; + sample_memory_ = 0; + prev_time_scale_ = false; + timescale_countdown_ = + tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1); + time_stretched_cn_samples_ = 0; + delay_manager_->Reset(); + buffer_level_filter_->Reset(); + packet_arrival_history_.Reset(); + last_playout_delay_ms_ = 0; +} + +void DecisionLogic::SetSampleRate(int fs_hz, size_t output_size_samples) { + // TODO(hlundin): Change to an enumerator and skip assert. + RTC_DCHECK(fs_hz == 8000 || fs_hz == 16000 || fs_hz == 32000 || + fs_hz == 48000); + sample_rate_khz_ = fs_hz / 1000; + output_size_samples_ = output_size_samples; + packet_arrival_history_.set_sample_rate(fs_hz); +} + +NetEq::Operation DecisionLogic::GetDecision(const NetEqStatus& status, + bool* reset_decoder) { + // If last mode was CNG (or Expand, since this could be covering up for + // a lost CNG packet), remember that CNG is on. This is needed if comfort + // noise is interrupted by DTMF. + if (status.last_mode == NetEq::Mode::kRfc3389Cng) { + cng_state_ = kCngRfc3389On; + } else if (status.last_mode == NetEq::Mode::kCodecInternalCng) { + cng_state_ = kCngInternalOn; + } + + if (IsExpand(status.last_mode)) { + ++num_consecutive_expands_; + } else { + num_consecutive_expands_ = 0; + } + + if (!IsExpand(status.last_mode) && !IsCng(status.last_mode)) { + last_playout_delay_ms_ = GetPlayoutDelayMs(status); + } + + prev_time_scale_ = prev_time_scale_ && IsTimestretch(status.last_mode); + if (prev_time_scale_) { + timescale_countdown_ = tick_timer_->GetNewCountdown(kMinTimescaleInterval); + } + if (!IsCng(status.last_mode)) { + FilterBufferLevel(status.packet_buffer_info.span_samples); + } + + // Guard for errors, to avoid getting stuck in error mode. + if (status.last_mode == NetEq::Mode::kError) { + if (!status.next_packet) { + return NetEq::Operation::kExpand; + } else { + // Use kUndefined to flag for a reset. + return NetEq::Operation::kUndefined; + } + } + + if (status.next_packet && status.next_packet->is_cng) { + return CngOperation(status); + } + + // Handle the case with no packet at all available (except maybe DTMF). + if (!status.next_packet) { + return NoPacket(status); + } + + // If the expand period was very long, reset NetEQ since it is likely that the + // sender was restarted. + if (num_consecutive_expands_ > config_.reinit_after_expands) { + *reset_decoder = true; + return NetEq::Operation::kNormal; + } + + // Make sure we don't restart audio too soon after an expansion to avoid + // running out of data right away again. We should only wait if there are no + // DTX or CNG packets in the buffer (otherwise we should just play out what we + // have, since we cannot know the exact duration of DTX or CNG packets), and + // if the mute factor is low enough (otherwise the expansion was short enough + // to not be noticable). + // Note that the MuteFactor is in Q14, so a value of 16384 corresponds to 1. + const int target_level_samples = TargetLevelMs() * sample_rate_khz_; + if (!config_.enable_stable_playout_delay && IsExpand(status.last_mode) && + status.expand_mutefactor < 16384 / 2 && + status.packet_buffer_info.span_samples < + static_cast(target_level_samples * kPostponeDecodingLevel / + 100) && + !status.packet_buffer_info.dtx_or_cng) { + return NetEq::Operation::kExpand; + } + + const uint32_t five_seconds_samples = + static_cast(5000 * sample_rate_khz_); + // Check if the required packet is available. + if (status.target_timestamp == status.next_packet->timestamp) { + return ExpectedPacketAvailable(status); + } + if (!PacketBuffer::IsObsoleteTimestamp(status.next_packet->timestamp, + status.target_timestamp, + five_seconds_samples)) { + return FuturePacketAvailable(status); + } + // This implies that available_timestamp < target_timestamp, which can + // happen when a new stream or codec is received. Signal for a reset. + return NetEq::Operation::kUndefined; +} + +void DecisionLogic::NotifyMutedState() { + ++num_consecutive_expands_; +} + +int DecisionLogic::TargetLevelMs() const { + int target_delay_ms = delay_manager_->TargetDelayMs(); + if (!config_.enable_stable_playout_delay) { + target_delay_ms = + std::max(target_delay_ms, + static_cast(packet_length_samples_ / sample_rate_khz_)); + } + return target_delay_ms; +} + +int DecisionLogic::UnlimitedTargetLevelMs() const { + return delay_manager_->UnlimitedTargetLevelMs(); +} + +int DecisionLogic::GetFilteredBufferLevel() const { + if (config_.enable_stable_playout_delay) { + return last_playout_delay_ms_ * sample_rate_khz_; + } + return buffer_level_filter_->filtered_current_level(); +} + +absl::optional DecisionLogic::PacketArrived( + int fs_hz, + bool should_update_stats, + const PacketArrivedInfo& info) { + buffer_flush_ = buffer_flush_ || info.buffer_flush; + if (!should_update_stats || info.is_cng_or_dtmf) { + return absl::nullopt; + } + if (info.packet_length_samples > 0 && fs_hz > 0 && + info.packet_length_samples != packet_length_samples_) { + packet_length_samples_ = info.packet_length_samples; + delay_manager_->SetPacketAudioLength(packet_length_samples_ * 1000 / fs_hz); + } + int64_t time_now_ms = tick_timer_->ticks() * tick_timer_->ms_per_tick(); + packet_arrival_history_.Insert(info.main_timestamp, time_now_ms); + if (packet_arrival_history_.size() < 2) { + // No meaningful delay estimate unless at least 2 packets have arrived. + return absl::nullopt; + } + int arrival_delay_ms = + packet_arrival_history_.GetDelayMs(info.main_timestamp, time_now_ms); + bool reordered = + !packet_arrival_history_.IsNewestRtpTimestamp(info.main_timestamp); + delay_manager_->Update(arrival_delay_ms, reordered); + return arrival_delay_ms; +} + +void DecisionLogic::FilterBufferLevel(size_t buffer_size_samples) { + buffer_level_filter_->SetTargetBufferLevel(TargetLevelMs()); + + int time_stretched_samples = time_stretched_cn_samples_; + if (prev_time_scale_) { + time_stretched_samples += sample_memory_; + } + + if (buffer_flush_) { + buffer_level_filter_->SetFilteredBufferLevel(buffer_size_samples); + buffer_flush_ = false; + } else { + buffer_level_filter_->Update(buffer_size_samples, time_stretched_samples); + } + prev_time_scale_ = false; + time_stretched_cn_samples_ = 0; +} + +NetEq::Operation DecisionLogic::CngOperation( + NetEqController::NetEqStatus status) { + // Signed difference between target and available timestamp. + int32_t timestamp_diff = static_cast( + static_cast(status.generated_noise_samples + + status.target_timestamp) - + status.next_packet->timestamp); + int optimal_level_samp = TargetLevelMs() * sample_rate_khz_; + const int64_t excess_waiting_time_samp = + -static_cast(timestamp_diff) - optimal_level_samp; + + if (excess_waiting_time_samp > optimal_level_samp / 2) { + // The waiting time for this packet will be longer than 1.5 + // times the wanted buffer delay. Apply fast-forward to cut the + // waiting time down to the optimal. + noise_fast_forward_ = rtc::saturated_cast(noise_fast_forward_ + + excess_waiting_time_samp); + timestamp_diff = + rtc::saturated_cast(timestamp_diff + excess_waiting_time_samp); + } + + if (timestamp_diff < 0 && status.last_mode == NetEq::Mode::kRfc3389Cng) { + // Not time to play this packet yet. Wait another round before using this + // packet. Keep on playing CNG from previous CNG parameters. + return NetEq::Operation::kRfc3389CngNoPacket; + } else { + // Otherwise, go for the CNG packet now. + noise_fast_forward_ = 0; + return NetEq::Operation::kRfc3389Cng; + } +} + +NetEq::Operation DecisionLogic::NoPacket(NetEqController::NetEqStatus status) { + if (cng_state_ == kCngRfc3389On) { + // Keep on playing comfort noise. + return NetEq::Operation::kRfc3389CngNoPacket; + } else if (cng_state_ == kCngInternalOn) { + // Stop CNG after a timeout. + if (config_.cng_timeout_ms && + status.generated_noise_samples > + static_cast(*config_.cng_timeout_ms * sample_rate_khz_)) { + return NetEq::Operation::kExpand; + } + return NetEq::Operation::kCodecInternalCng; + } else if (status.play_dtmf) { + return NetEq::Operation::kDtmf; + } else { + // Nothing to play, do expand. + return NetEq::Operation::kExpand; + } +} + +NetEq::Operation DecisionLogic::ExpectedPacketAvailable( + NetEqController::NetEqStatus status) { + if (!disallow_time_stretching_ && status.last_mode != NetEq::Mode::kExpand && + !status.play_dtmf) { + if (config_.enable_stable_playout_delay) { + const int playout_delay_ms = GetPlayoutDelayMs(status); + if (playout_delay_ms >= HighThreshold() << 2) { + return NetEq::Operation::kFastAccelerate; + } + if (TimescaleAllowed()) { + if (playout_delay_ms >= HighThreshold()) { + return NetEq::Operation::kAccelerate; + } + if (playout_delay_ms < LowThreshold()) { + return NetEq::Operation::kPreemptiveExpand; + } + } + } else { + const int target_level_samples = TargetLevelMs() * sample_rate_khz_; + const int low_limit = std::max( + target_level_samples * 3 / 4, + target_level_samples - + config_.deceleration_target_level_offset_ms * sample_rate_khz_); + const int high_limit = std::max( + target_level_samples, + low_limit + kDelayAdjustmentGranularityMs * sample_rate_khz_); + + const int buffer_level_samples = + buffer_level_filter_->filtered_current_level(); + if (buffer_level_samples >= high_limit << 2) + return NetEq::Operation::kFastAccelerate; + if (TimescaleAllowed()) { + if (buffer_level_samples >= high_limit) + return NetEq::Operation::kAccelerate; + if (buffer_level_samples < low_limit) + return NetEq::Operation::kPreemptiveExpand; + } + } + } + return NetEq::Operation::kNormal; +} + +NetEq::Operation DecisionLogic::FuturePacketAvailable( + NetEqController::NetEqStatus status) { + // Required packet is not available, but a future packet is. + // Check if we should continue with an ongoing expand because the new packet + // is too far into the future. + if (IsExpand(status.last_mode) && ShouldContinueExpand(status)) { + if (status.play_dtmf) { + // Still have DTMF to play, so do not do expand. + return NetEq::Operation::kDtmf; + } else { + // Nothing to play. + return NetEq::Operation::kExpand; + } + } + + if (status.last_mode == NetEq::Mode::kCodecPlc) { + return NetEq::Operation::kNormal; + } + + // If previous was comfort noise, then no merge is needed. + if (IsCng(status.last_mode)) { + uint32_t timestamp_leap = + status.next_packet->timestamp - status.target_timestamp; + const bool generated_enough_noise = + status.generated_noise_samples >= timestamp_leap; + + int playout_delay_ms = GetNextPacketDelayMs(status); + const bool above_target_delay = playout_delay_ms > HighThresholdCng(); + const bool below_target_delay = playout_delay_ms < LowThresholdCng(); + // Keep the delay same as before CNG, but make sure that it is within the + // target window. + if ((generated_enough_noise && !below_target_delay) || above_target_delay) { + time_stretched_cn_samples_ = + timestamp_leap - status.generated_noise_samples; + return NetEq::Operation::kNormal; + } + + if (status.last_mode == NetEq::Mode::kRfc3389Cng) { + return NetEq::Operation::kRfc3389CngNoPacket; + } + return NetEq::Operation::kCodecInternalCng; + } + + // Do not merge unless we have done an expand before. + if (status.last_mode == NetEq::Mode::kExpand) { + return NetEq::Operation::kMerge; + } else if (status.play_dtmf) { + // Play DTMF instead of expand. + return NetEq::Operation::kDtmf; + } else { + return NetEq::Operation::kExpand; + } +} + +bool DecisionLogic::UnderTargetLevel() const { + return buffer_level_filter_->filtered_current_level() < + TargetLevelMs() * sample_rate_khz_; +} + +bool DecisionLogic::ReinitAfterExpands(uint32_t timestamp_leap) const { + return timestamp_leap >= static_cast(output_size_samples_ * + config_.reinit_after_expands); +} + +bool DecisionLogic::PacketTooEarly(uint32_t timestamp_leap) const { + return timestamp_leap > + static_cast(output_size_samples_ * num_consecutive_expands_); +} + +bool DecisionLogic::MaxWaitForPacket() const { + return num_consecutive_expands_ >= kMaxWaitForPacketTicks; +} + +bool DecisionLogic::ShouldContinueExpand( + NetEqController::NetEqStatus status) const { + uint32_t timestamp_leap = + status.next_packet->timestamp - status.target_timestamp; + if (config_.enable_stable_playout_delay) { + return GetNextPacketDelayMs(status) < HighThreshold() && + PacketTooEarly(timestamp_leap); + } + return !ReinitAfterExpands(timestamp_leap) && !MaxWaitForPacket() && + PacketTooEarly(timestamp_leap) && UnderTargetLevel(); +} + +int DecisionLogic::GetNextPacketDelayMs( + NetEqController::NetEqStatus status) const { + if (config_.enable_stable_playout_delay) { + return packet_arrival_history_.GetDelayMs( + status.next_packet->timestamp, + tick_timer_->ticks() * tick_timer_->ms_per_tick()); + } + return status.packet_buffer_info.span_samples / sample_rate_khz_; +} + +int DecisionLogic::GetPlayoutDelayMs( + NetEqController::NetEqStatus status) const { + uint32_t playout_timestamp = + status.target_timestamp - status.sync_buffer_samples; + return packet_arrival_history_.GetDelayMs( + playout_timestamp, tick_timer_->ticks() * tick_timer_->ms_per_tick()); +} + +int DecisionLogic::LowThreshold() const { + int target_delay_ms = TargetLevelMs(); + return std::max( + target_delay_ms * 3 / 4, + target_delay_ms - config_.deceleration_target_level_offset_ms); +} + +int DecisionLogic::HighThreshold() const { + if (config_.enable_stable_playout_delay) { + return std::max(TargetLevelMs(), packet_arrival_history_.GetMaxDelayMs()) + + kDelayAdjustmentGranularityMs; + } + return std::max(TargetLevelMs(), + LowThreshold() + kDelayAdjustmentGranularityMs); +} + +int DecisionLogic::LowThresholdCng() const { + if (config_.enable_stable_playout_delay) { + return LowThreshold(); + } + return std::max(0, TargetLevelMs() - kTargetLevelWindowMs / 2); +} + +int DecisionLogic::HighThresholdCng() const { + if (config_.enable_stable_playout_delay) { + return HighThreshold(); + } + return TargetLevelMs() + kTargetLevelWindowMs / 2; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.h b/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.h new file mode 100644 index 0000000000..8d1ff4d622 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.h @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_ +#define MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_ + +#include + +#include "api/neteq/neteq.h" +#include "api/neteq/neteq_controller.h" +#include "api/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/buffer_level_filter.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "modules/audio_coding/neteq/packet_arrival_history.h" +#include "rtc_base/experiments/field_trial_parser.h" + +namespace webrtc { + +// This is the class for the decision tree implementation. +class DecisionLogic : public NetEqController { + public: + DecisionLogic(NetEqController::Config config); + DecisionLogic(NetEqController::Config config, + std::unique_ptr delay_manager, + std::unique_ptr buffer_level_filter); + + ~DecisionLogic() override; + + DecisionLogic(const DecisionLogic&) = delete; + DecisionLogic& operator=(const DecisionLogic&) = delete; + + // Not used. + void Reset() override {} + + // Resets parts of the state. Typically done when switching codecs. + void SoftReset() override; + + // Sets the sample rate and the output block size. + void SetSampleRate(int fs_hz, size_t output_size_samples) override; + + // Given info about the latest received packet, and current jitter buffer + // status, returns the operation. `target_timestamp` and `expand_mutefactor` + // are provided for reference. `last_packet_samples` is the number of samples + // obtained from the last decoded frame. If there is a packet available, it + // should be supplied in `packet`; otherwise it should be NULL. The mode + // resulting from the last call to NetEqImpl::GetAudio is supplied in + // `last_mode`. If there is a DTMF event to play, `play_dtmf` should be set to + // true. The output variable `reset_decoder` will be set to true if a reset is + // required; otherwise it is left unchanged (i.e., it can remain true if it + // was true before the call). + NetEq::Operation GetDecision(const NetEqController::NetEqStatus& status, + bool* reset_decoder) override; + + // These methods test the `cng_state_` for different conditions. + bool CngRfc3389On() const override { return cng_state_ == kCngRfc3389On; } + bool CngOff() const override { return cng_state_ == kCngOff; } + + // Resets the `cng_state_` to kCngOff. + void SetCngOff() override { cng_state_ = kCngOff; } + + void ExpandDecision(NetEq::Operation operation) override {} + + // Adds `value` to `sample_memory_`. + void AddSampleMemory(int32_t value) override { sample_memory_ += value; } + + int TargetLevelMs() const override; + + int UnlimitedTargetLevelMs() const override; + + absl::optional PacketArrived(int fs_hz, + bool should_update_stats, + const PacketArrivedInfo& info) override; + + void RegisterEmptyPacket() override {} + + void NotifyMutedState() override; + + bool SetMaximumDelay(int delay_ms) override { + return delay_manager_->SetMaximumDelay(delay_ms); + } + bool SetMinimumDelay(int delay_ms) override { + return delay_manager_->SetMinimumDelay(delay_ms); + } + bool SetBaseMinimumDelay(int delay_ms) override { + return delay_manager_->SetBaseMinimumDelay(delay_ms); + } + int GetBaseMinimumDelay() const override { + return delay_manager_->GetBaseMinimumDelay(); + } + bool PeakFound() const override { return false; } + + int GetFilteredBufferLevel() const override; + + // Accessors and mutators. + void set_sample_memory(int32_t value) override { sample_memory_ = value; } + size_t noise_fast_forward() const override { return noise_fast_forward_; } + size_t packet_length_samples() const override { + return packet_length_samples_; + } + void set_packet_length_samples(size_t value) override { + packet_length_samples_ = value; + } + void set_prev_time_scale(bool value) override { prev_time_scale_ = value; } + + private: + // The value 5 sets maximum time-stretch rate to about 100 ms/s. + static const int kMinTimescaleInterval = 5; + + enum CngState { kCngOff, kCngRfc3389On, kCngInternalOn }; + + // Updates the `buffer_level_filter_` with the current buffer level + // `buffer_size_samples`. + void FilterBufferLevel(size_t buffer_size_samples); + + // Returns the operation given that the next available packet is a comfort + // noise payload (RFC 3389 only, not codec-internal). + virtual NetEq::Operation CngOperation(NetEqController::NetEqStatus status); + + // Returns the operation given that no packets are available (except maybe + // a DTMF event, flagged by setting `play_dtmf` true). + virtual NetEq::Operation NoPacket(NetEqController::NetEqStatus status); + + // Returns the operation to do given that the expected packet is available. + virtual NetEq::Operation ExpectedPacketAvailable( + NetEqController::NetEqStatus status); + + // Returns the operation to do given that the expected packet is not + // available, but a packet further into the future is at hand. + virtual NetEq::Operation FuturePacketAvailable( + NetEqController::NetEqStatus status); + + // Checks if enough time has elapsed since the last successful timescale + // operation was done (i.e., accelerate or preemptive expand). + bool TimescaleAllowed() const { + return !timescale_countdown_ || timescale_countdown_->Finished(); + } + + // Checks if the current (filtered) buffer level is under the target level. + bool UnderTargetLevel() const; + + // Checks if `timestamp_leap` is so long into the future that a reset due + // to exceeding kReinitAfterExpands will be done. + bool ReinitAfterExpands(uint32_t timestamp_leap) const; + + // Checks if we still have not done enough expands to cover the distance from + // the last decoded packet to the next available packet, the distance beeing + // conveyed in `timestamp_leap`. + bool PacketTooEarly(uint32_t timestamp_leap) const; + + bool MaxWaitForPacket() const; + + bool ShouldContinueExpand(NetEqController::NetEqStatus status) const; + + int GetNextPacketDelayMs(NetEqController::NetEqStatus status) const; + int GetPlayoutDelayMs(NetEqController::NetEqStatus status) const; + + int LowThreshold() const; + int HighThreshold() const; + int LowThresholdCng() const; + int HighThresholdCng() const; + + // Runtime configurable options through field trial + // WebRTC-Audio-NetEqDecisionLogicConfig. + struct Config { + Config(); + + bool enable_stable_playout_delay = false; + int reinit_after_expands = 100; + int deceleration_target_level_offset_ms = 85; + int packet_history_size_ms = 2000; + absl::optional cng_timeout_ms; + }; + Config config_; + std::unique_ptr delay_manager_; + std::unique_ptr buffer_level_filter_; + PacketArrivalHistory packet_arrival_history_; + const TickTimer* tick_timer_; + int sample_rate_khz_; + size_t output_size_samples_; + CngState cng_state_ = kCngOff; // Remember if comfort noise is interrupted by + // other event (e.g., DTMF). + size_t noise_fast_forward_ = 0; + size_t packet_length_samples_ = 0; + int sample_memory_ = 0; + bool prev_time_scale_ = false; + bool disallow_time_stretching_; + std::unique_ptr timescale_countdown_; + int num_consecutive_expands_ = 0; + int time_stretched_cn_samples_ = 0; + bool buffer_flush_ = false; + int last_playout_delay_ms_ = 0; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic_unittest.cc new file mode 100644 index 0000000000..6150c9a6db --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic_unittest.cc @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for DecisionLogic class and derived classes. + +#include "modules/audio_coding/neteq/decision_logic.h" + +#include "api/neteq/neteq_controller.h" +#include "api/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/buffer_level_filter.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "modules/audio_coding/neteq/mock/mock_buffer_level_filter.h" +#include "modules/audio_coding/neteq/mock/mock_delay_manager.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kSampleRate = 8000; +constexpr int kSamplesPerMs = kSampleRate / 1000; +constexpr int kOutputSizeSamples = kSamplesPerMs * 10; +constexpr int kMinTimescaleInterval = 5; + +NetEqController::NetEqStatus CreateNetEqStatus(NetEq::Mode last_mode, + int current_delay_ms) { + NetEqController::NetEqStatus status; + status.play_dtmf = false; + status.last_mode = last_mode; + status.target_timestamp = 1234; + status.generated_noise_samples = 0; + status.expand_mutefactor = 0; + status.packet_buffer_info.num_samples = current_delay_ms * kSamplesPerMs; + status.packet_buffer_info.span_samples = current_delay_ms * kSamplesPerMs; + status.packet_buffer_info.span_samples_no_dtx = + current_delay_ms * kSamplesPerMs; + status.packet_buffer_info.dtx_or_cng = false; + status.next_packet = {status.target_timestamp, false, false}; + return status; +} + +using ::testing::Return; + +} // namespace + +class DecisionLogicTest : public ::testing::Test { + protected: + DecisionLogicTest() { + test::ScopedFieldTrials trials( + "WebRTC-Audio-NetEqDecisionLogicConfig/cng_timeout_ms:1000/"); + NetEqController::Config config; + config.tick_timer = &tick_timer_; + config.allow_time_stretching = true; + auto delay_manager = std::make_unique( + DelayManager::Config(), config.tick_timer); + mock_delay_manager_ = delay_manager.get(); + auto buffer_level_filter = std::make_unique(); + mock_buffer_level_filter_ = buffer_level_filter.get(); + decision_logic_ = std::make_unique( + config, std::move(delay_manager), std::move(buffer_level_filter)); + decision_logic_->SetSampleRate(kSampleRate, kOutputSizeSamples); + } + + TickTimer tick_timer_; + std::unique_ptr decision_logic_; + MockDelayManager* mock_delay_manager_; + MockBufferLevelFilter* mock_buffer_level_filter_; +}; + +TEST_F(DecisionLogicTest, NormalOperation) { + EXPECT_CALL(*mock_delay_manager_, TargetDelayMs()) + .WillRepeatedly(Return(100)); + EXPECT_CALL(*mock_buffer_level_filter_, filtered_current_level()) + .WillRepeatedly(Return(90 * kSamplesPerMs)); + + bool reset_decoder = false; + tick_timer_.Increment(kMinTimescaleInterval + 1); + EXPECT_EQ(decision_logic_->GetDecision( + CreateNetEqStatus(NetEq::Mode::kNormal, 100), &reset_decoder), + NetEq::Operation::kNormal); + EXPECT_FALSE(reset_decoder); +} + +TEST_F(DecisionLogicTest, Accelerate) { + EXPECT_CALL(*mock_delay_manager_, TargetDelayMs()) + .WillRepeatedly(Return(100)); + EXPECT_CALL(*mock_buffer_level_filter_, filtered_current_level()) + .WillRepeatedly(Return(110 * kSamplesPerMs)); + + bool reset_decoder = false; + tick_timer_.Increment(kMinTimescaleInterval + 1); + EXPECT_EQ(decision_logic_->GetDecision( + CreateNetEqStatus(NetEq::Mode::kNormal, 100), &reset_decoder), + NetEq::Operation::kAccelerate); + EXPECT_FALSE(reset_decoder); +} + +TEST_F(DecisionLogicTest, FastAccelerate) { + EXPECT_CALL(*mock_delay_manager_, TargetDelayMs()) + .WillRepeatedly(Return(100)); + EXPECT_CALL(*mock_buffer_level_filter_, filtered_current_level()) + .WillRepeatedly(Return(400 * kSamplesPerMs)); + + bool reset_decoder = false; + tick_timer_.Increment(kMinTimescaleInterval + 1); + EXPECT_EQ(decision_logic_->GetDecision( + CreateNetEqStatus(NetEq::Mode::kNormal, 100), &reset_decoder), + NetEq::Operation::kFastAccelerate); + EXPECT_FALSE(reset_decoder); +} + +TEST_F(DecisionLogicTest, PreemptiveExpand) { + EXPECT_CALL(*mock_delay_manager_, TargetDelayMs()) + .WillRepeatedly(Return(100)); + EXPECT_CALL(*mock_buffer_level_filter_, filtered_current_level()) + .WillRepeatedly(Return(50 * kSamplesPerMs)); + + bool reset_decoder = false; + tick_timer_.Increment(kMinTimescaleInterval + 1); + EXPECT_EQ(decision_logic_->GetDecision( + CreateNetEqStatus(NetEq::Mode::kNormal, 100), &reset_decoder), + NetEq::Operation::kPreemptiveExpand); + EXPECT_FALSE(reset_decoder); +} + +TEST_F(DecisionLogicTest, DecelerationTargetLevelOffset) { + EXPECT_CALL(*mock_delay_manager_, TargetDelayMs()) + .WillRepeatedly(Return(500)); + EXPECT_CALL(*mock_buffer_level_filter_, filtered_current_level()) + .WillRepeatedly(Return(400 * kSamplesPerMs)); + + bool reset_decoder = false; + tick_timer_.Increment(kMinTimescaleInterval + 1); + EXPECT_EQ(decision_logic_->GetDecision( + CreateNetEqStatus(NetEq::Mode::kNormal, 400), &reset_decoder), + NetEq::Operation::kPreemptiveExpand); + EXPECT_FALSE(reset_decoder); +} + +TEST_F(DecisionLogicTest, PostponeDecodeAfterExpand) { + EXPECT_CALL(*mock_delay_manager_, TargetDelayMs()) + .WillRepeatedly(Return(500)); + + // Below 50% target delay threshold. + bool reset_decoder = false; + EXPECT_EQ(decision_logic_->GetDecision( + CreateNetEqStatus(NetEq::Mode::kExpand, 200), &reset_decoder), + NetEq::Operation::kExpand); + EXPECT_FALSE(reset_decoder); + + // Above 50% target delay threshold. + EXPECT_EQ(decision_logic_->GetDecision( + CreateNetEqStatus(NetEq::Mode::kExpand, 250), &reset_decoder), + NetEq::Operation::kNormal); + EXPECT_FALSE(reset_decoder); +} + +TEST_F(DecisionLogicTest, TimeStrechComfortNoise) { + EXPECT_CALL(*mock_delay_manager_, TargetDelayMs()) + .WillRepeatedly(Return(500)); + + { + bool reset_decoder = false; + // Below target window. + auto status = CreateNetEqStatus(NetEq::Mode::kCodecInternalCng, 400); + status.generated_noise_samples = 400 * kSamplesPerMs; + status.next_packet->timestamp = + status.target_timestamp + 400 * kSamplesPerMs; + EXPECT_EQ(decision_logic_->GetDecision(status, &reset_decoder), + NetEq::Operation::kCodecInternalCng); + EXPECT_FALSE(reset_decoder); + } + + { + bool reset_decoder = false; + // Above target window. + auto status = CreateNetEqStatus(NetEq::Mode::kCodecInternalCng, 600); + status.generated_noise_samples = 200 * kSamplesPerMs; + status.next_packet->timestamp = + status.target_timestamp + 400 * kSamplesPerMs; + EXPECT_EQ(decision_logic_->GetDecision(status, &reset_decoder), + NetEq::Operation::kNormal); + EXPECT_FALSE(reset_decoder); + + // The buffer level filter should be adjusted with the number of samples + // that was skipped. + int timestamp_leap = status.next_packet->timestamp - + status.target_timestamp - + status.generated_noise_samples; + EXPECT_CALL(*mock_buffer_level_filter_, + Update(400 * kSamplesPerMs, timestamp_leap)); + EXPECT_EQ(decision_logic_->GetDecision( + CreateNetEqStatus(NetEq::Mode::kNormal, 400), &reset_decoder), + NetEq::Operation::kNormal); + EXPECT_FALSE(reset_decoder); + } +} + +TEST_F(DecisionLogicTest, CngTimeout) { + auto status = CreateNetEqStatus(NetEq::Mode::kCodecInternalCng, 0); + status.next_packet = absl::nullopt; + status.generated_noise_samples = kSamplesPerMs * 500; + bool reset_decoder = false; + EXPECT_EQ(decision_logic_->GetDecision(status, &reset_decoder), + NetEq::Operation::kCodecInternalCng); + status.generated_noise_samples = kSamplesPerMs * 1010; + EXPECT_EQ(decision_logic_->GetDecision(status, &reset_decoder), + NetEq::Operation::kExpand); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.cc b/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.cc new file mode 100644 index 0000000000..3447ced1da --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.cc @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/decoder_database.h" + +#include + +#include +#include +#include +#include + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "api/audio_codecs/audio_decoder.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/audio_format_to_string.h" + +namespace webrtc { + +DecoderDatabase::DecoderDatabase( + const rtc::scoped_refptr& decoder_factory, + absl::optional codec_pair_id) + : active_decoder_type_(-1), + active_cng_decoder_type_(-1), + decoder_factory_(decoder_factory), + codec_pair_id_(codec_pair_id) {} + +DecoderDatabase::~DecoderDatabase() = default; + +DecoderDatabase::DecoderInfo::DecoderInfo( + const SdpAudioFormat& audio_format, + absl::optional codec_pair_id, + AudioDecoderFactory* factory, + absl::string_view codec_name) + : name_(codec_name), + audio_format_(audio_format), + codec_pair_id_(codec_pair_id), + factory_(factory), + cng_decoder_(CngDecoder::Create(audio_format)), + subtype_(SubtypeFromFormat(audio_format)) {} + +DecoderDatabase::DecoderInfo::DecoderInfo( + const SdpAudioFormat& audio_format, + absl::optional codec_pair_id, + AudioDecoderFactory* factory) + : DecoderInfo(audio_format, codec_pair_id, factory, audio_format.name) {} + +DecoderDatabase::DecoderInfo::DecoderInfo(DecoderInfo&&) = default; +DecoderDatabase::DecoderInfo::~DecoderInfo() = default; + +AudioDecoder* DecoderDatabase::DecoderInfo::GetDecoder() const { + if (subtype_ != Subtype::kNormal) { + // These are handled internally, so they have no AudioDecoder objects. + return nullptr; + } + if (!decoder_) { + // TODO(ossu): Keep a check here for now, since a number of tests create + // DecoderInfos without factories. + RTC_DCHECK(factory_); + decoder_ = factory_->MakeAudioDecoder(audio_format_, codec_pair_id_); + } + RTC_DCHECK(decoder_) << "Failed to create: " << rtc::ToString(audio_format_); + return decoder_.get(); +} + +bool DecoderDatabase::DecoderInfo::IsType(absl::string_view name) const { + return absl::EqualsIgnoreCase(audio_format_.name, name); +} + +absl::optional +DecoderDatabase::DecoderInfo::CngDecoder::Create(const SdpAudioFormat& format) { + if (absl::EqualsIgnoreCase(format.name, "CN")) { + // CN has a 1:1 RTP clock rate to sample rate ratio. + const int sample_rate_hz = format.clockrate_hz; + RTC_DCHECK(sample_rate_hz == 8000 || sample_rate_hz == 16000 || + sample_rate_hz == 32000 || sample_rate_hz == 48000); + return DecoderDatabase::DecoderInfo::CngDecoder{sample_rate_hz}; + } else { + return absl::nullopt; + } +} + +DecoderDatabase::DecoderInfo::Subtype +DecoderDatabase::DecoderInfo::SubtypeFromFormat(const SdpAudioFormat& format) { + if (absl::EqualsIgnoreCase(format.name, "CN")) { + return Subtype::kComfortNoise; + } else if (absl::EqualsIgnoreCase(format.name, "telephone-event")) { + return Subtype::kDtmf; + } else if (absl::EqualsIgnoreCase(format.name, "red")) { + return Subtype::kRed; + } + + return Subtype::kNormal; +} + +bool DecoderDatabase::Empty() const { + return decoders_.empty(); +} + +int DecoderDatabase::Size() const { + return static_cast(decoders_.size()); +} + +std::vector DecoderDatabase::SetCodecs( + const std::map& codecs) { + // First collect all payload types that we'll remove or reassign, then remove + // them from the database. + std::vector changed_payload_types; + for (const std::pair kv : decoders_) { + auto i = codecs.find(kv.first); + if (i == codecs.end() || i->second != kv.second.GetFormat()) { + changed_payload_types.push_back(kv.first); + } + } + for (int pl_type : changed_payload_types) { + Remove(pl_type); + } + + // Enter the new and changed payload type mappings into the database. + for (const auto& kv : codecs) { + const int& rtp_payload_type = kv.first; + const SdpAudioFormat& audio_format = kv.second; + RTC_DCHECK_GE(rtp_payload_type, 0); + RTC_DCHECK_LE(rtp_payload_type, 0x7f); + if (decoders_.count(rtp_payload_type) == 0) { + decoders_.insert(std::make_pair( + rtp_payload_type, + DecoderInfo(audio_format, codec_pair_id_, decoder_factory_.get()))); + } else { + // The mapping for this payload type hasn't changed. + } + } + + return changed_payload_types; +} + +int DecoderDatabase::RegisterPayload(int rtp_payload_type, + const SdpAudioFormat& audio_format) { + if (rtp_payload_type < 0 || rtp_payload_type > 0x7f) { + return kInvalidRtpPayloadType; + } + const auto ret = decoders_.insert(std::make_pair( + rtp_payload_type, + DecoderInfo(audio_format, codec_pair_id_, decoder_factory_.get()))); + if (ret.second == false) { + // Database already contains a decoder with type `rtp_payload_type`. + return kDecoderExists; + } + return kOK; +} + +int DecoderDatabase::Remove(uint8_t rtp_payload_type) { + if (decoders_.erase(rtp_payload_type) == 0) { + // No decoder with that `rtp_payload_type`. + return kDecoderNotFound; + } + if (active_decoder_type_ == rtp_payload_type) { + active_decoder_type_ = -1; // No active decoder. + } + if (active_cng_decoder_type_ == rtp_payload_type) { + active_cng_decoder_type_ = -1; // No active CNG decoder. + } + return kOK; +} + +void DecoderDatabase::RemoveAll() { + decoders_.clear(); + active_decoder_type_ = -1; // No active decoder. + active_cng_decoder_type_ = -1; // No active CNG decoder. +} + +const DecoderDatabase::DecoderInfo* DecoderDatabase::GetDecoderInfo( + uint8_t rtp_payload_type) const { + DecoderMap::const_iterator it = decoders_.find(rtp_payload_type); + if (it == decoders_.end()) { + // Decoder not found. + return NULL; + } + return &it->second; +} + +int DecoderDatabase::SetActiveDecoder(uint8_t rtp_payload_type, + bool* new_decoder) { + // Check that `rtp_payload_type` exists in the database. + const DecoderInfo* info = GetDecoderInfo(rtp_payload_type); + if (!info) { + // Decoder not found. + return kDecoderNotFound; + } + RTC_CHECK(!info->IsComfortNoise()); + RTC_DCHECK(new_decoder); + *new_decoder = false; + if (active_decoder_type_ < 0) { + // This is the first active decoder. + *new_decoder = true; + } else if (active_decoder_type_ != rtp_payload_type) { + // Moving from one active decoder to another. Delete the first one. + const DecoderInfo* old_info = GetDecoderInfo(active_decoder_type_); + RTC_DCHECK(old_info); + old_info->DropDecoder(); + *new_decoder = true; + } + active_decoder_type_ = rtp_payload_type; + return kOK; +} + +AudioDecoder* DecoderDatabase::GetActiveDecoder() const { + if (active_decoder_type_ < 0) { + // No active decoder. + return NULL; + } + return GetDecoder(active_decoder_type_); +} + +int DecoderDatabase::SetActiveCngDecoder(uint8_t rtp_payload_type) { + // Check that `rtp_payload_type` exists in the database. + const DecoderInfo* info = GetDecoderInfo(rtp_payload_type); + if (!info) { + // Decoder not found. + return kDecoderNotFound; + } + if (active_cng_decoder_type_ >= 0 && + active_cng_decoder_type_ != rtp_payload_type) { + // Moving from one active CNG decoder to another. Delete the first one. + RTC_DCHECK(active_cng_decoder_); + active_cng_decoder_.reset(); + } + active_cng_decoder_type_ = rtp_payload_type; + return kOK; +} + +ComfortNoiseDecoder* DecoderDatabase::GetActiveCngDecoder() const { + if (active_cng_decoder_type_ < 0) { + // No active CNG decoder. + return NULL; + } + if (!active_cng_decoder_) { + active_cng_decoder_.reset(new ComfortNoiseDecoder); + } + return active_cng_decoder_.get(); +} + +AudioDecoder* DecoderDatabase::GetDecoder(uint8_t rtp_payload_type) const { + const DecoderInfo* info = GetDecoderInfo(rtp_payload_type); + return info ? info->GetDecoder() : nullptr; +} + +bool DecoderDatabase::IsComfortNoise(uint8_t rtp_payload_type) const { + const DecoderInfo* info = GetDecoderInfo(rtp_payload_type); + return info && info->IsComfortNoise(); +} + +bool DecoderDatabase::IsDtmf(uint8_t rtp_payload_type) const { + const DecoderInfo* info = GetDecoderInfo(rtp_payload_type); + return info && info->IsDtmf(); +} + +bool DecoderDatabase::IsRed(uint8_t rtp_payload_type) const { + const DecoderInfo* info = GetDecoderInfo(rtp_payload_type); + return info && info->IsRed(); +} + +int DecoderDatabase::CheckPayloadTypes(const PacketList& packet_list) const { + PacketList::const_iterator it; + for (it = packet_list.begin(); it != packet_list.end(); ++it) { + if (!GetDecoderInfo(it->payload_type)) { + // Payload type is not found. + RTC_LOG(LS_WARNING) << "CheckPayloadTypes: unknown RTP payload type " + << static_cast(it->payload_type); + return kDecoderNotFound; + } + } + return kOK; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.h b/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.h new file mode 100644 index 0000000000..8cf2019135 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.h @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DECODER_DATABASE_H_ +#define MODULES_AUDIO_CODING_NETEQ_DECODER_DATABASE_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_format.h" +#include "api/scoped_refptr.h" +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" +#include "modules/audio_coding/neteq/packet.h" + +namespace webrtc { + +class DecoderDatabase { + public: + enum DatabaseReturnCodes { + kOK = 0, + kInvalidRtpPayloadType = -1, + kCodecNotSupported = -2, + kInvalidSampleRate = -3, + kDecoderExists = -4, + kDecoderNotFound = -5, + kInvalidPointer = -6 + }; + + // Class that stores decoder info in the database. + class DecoderInfo { + public: + DecoderInfo(const SdpAudioFormat& audio_format, + absl::optional codec_pair_id, + AudioDecoderFactory* factory, + absl::string_view codec_name); + explicit DecoderInfo(const SdpAudioFormat& audio_format, + absl::optional codec_pair_id, + AudioDecoderFactory* factory = nullptr); + DecoderInfo(DecoderInfo&&); + ~DecoderInfo(); + + // Get the AudioDecoder object, creating it first if necessary. + AudioDecoder* GetDecoder() const; + + // Delete the AudioDecoder object, unless it's external. (This means we can + // always recreate it later if we need it.) + void DropDecoder() const { decoder_.reset(); } + + int SampleRateHz() const { + if (IsDtmf()) { + // DTMF has a 1:1 mapping between clock rate and sample rate. + return audio_format_.clockrate_hz; + } + const AudioDecoder* decoder = GetDecoder(); + RTC_DCHECK_EQ(1, !!decoder + !!cng_decoder_); + return decoder ? decoder->SampleRateHz() : cng_decoder_->sample_rate_hz; + } + + const SdpAudioFormat& GetFormat() const { return audio_format_; } + + // Returns true if the decoder's format is comfort noise. + bool IsComfortNoise() const { + RTC_DCHECK_EQ(!!cng_decoder_, subtype_ == Subtype::kComfortNoise); + return subtype_ == Subtype::kComfortNoise; + } + + // Returns true if the decoder's format is DTMF. + bool IsDtmf() const { return subtype_ == Subtype::kDtmf; } + + // Returns true if the decoder's format is RED. + bool IsRed() const { return subtype_ == Subtype::kRed; } + + // Returns true if the decoder's format is named `name`. + bool IsType(absl::string_view name) const; + + const std::string& get_name() const { return name_; } + + private: + // TODO(ossu): `name_` is kept here while we retain the old external + // decoder interface. Remove this once using an + // AudioDecoderFactory has supplanted the old functionality. + const std::string name_; + + const SdpAudioFormat audio_format_; + const absl::optional codec_pair_id_; + AudioDecoderFactory* const factory_; + mutable std::unique_ptr decoder_; + + // Set iff this is a comfort noise decoder. + struct CngDecoder { + static absl::optional Create(const SdpAudioFormat& format); + int sample_rate_hz; + }; + const absl::optional cng_decoder_; + + enum class Subtype : int8_t { kNormal, kComfortNoise, kDtmf, kRed }; + + static Subtype SubtypeFromFormat(const SdpAudioFormat& format); + + const Subtype subtype_; + }; + + // Maximum value for 8 bits, and an invalid RTP payload type (since it is + // only 7 bits). + static const uint8_t kRtpPayloadTypeError = 0xFF; + + DecoderDatabase( + const rtc::scoped_refptr& decoder_factory, + absl::optional codec_pair_id); + + virtual ~DecoderDatabase(); + + DecoderDatabase(const DecoderDatabase&) = delete; + DecoderDatabase& operator=(const DecoderDatabase&) = delete; + + // Returns true if the database is empty. + virtual bool Empty() const; + + // Returns the number of decoders registered in the database. + virtual int Size() const; + + // Replaces the existing set of decoders with the given set. Returns the + // payload types that were reassigned or removed while doing so. + virtual std::vector SetCodecs( + const std::map& codecs); + + // Registers a decoder for the given payload type. Returns kOK on success; + // otherwise an error code. + virtual int RegisterPayload(int rtp_payload_type, + const SdpAudioFormat& audio_format); + + // Removes the entry for `rtp_payload_type` from the database. + // Returns kDecoderNotFound or kOK depending on the outcome of the operation. + virtual int Remove(uint8_t rtp_payload_type); + + // Remove all entries. + virtual void RemoveAll(); + + // Returns a pointer to the DecoderInfo struct for `rtp_payload_type`. If + // no decoder is registered with that `rtp_payload_type`, NULL is returned. + virtual const DecoderInfo* GetDecoderInfo(uint8_t rtp_payload_type) const; + + // Sets the active decoder to be `rtp_payload_type`. If this call results in a + // change of active decoder, `new_decoder` is set to true. The previous active + // decoder's AudioDecoder object is deleted. + virtual int SetActiveDecoder(uint8_t rtp_payload_type, bool* new_decoder); + + // Returns the current active decoder, or NULL if no active decoder exists. + virtual AudioDecoder* GetActiveDecoder() const; + + // Sets the active comfort noise decoder to be `rtp_payload_type`. If this + // call results in a change of active comfort noise decoder, the previous + // active decoder's AudioDecoder object is deleted. + virtual int SetActiveCngDecoder(uint8_t rtp_payload_type); + + // Returns the current active comfort noise decoder, or NULL if no active + // comfort noise decoder exists. + virtual ComfortNoiseDecoder* GetActiveCngDecoder() const; + + // The following are utility methods: they will look up DecoderInfo through + // GetDecoderInfo and call the respective method on that info object, if it + // exists. + + // Returns a pointer to the AudioDecoder object associated with + // `rtp_payload_type`, or NULL if none is registered. If the AudioDecoder + // object does not exist for that decoder, the object is created. + AudioDecoder* GetDecoder(uint8_t rtp_payload_type) const; + + // Returns true if `rtp_payload_type` is registered as comfort noise. + bool IsComfortNoise(uint8_t rtp_payload_type) const; + + // Returns true if `rtp_payload_type` is registered as DTMF. + bool IsDtmf(uint8_t rtp_payload_type) const; + + // Returns true if `rtp_payload_type` is registered as RED. + bool IsRed(uint8_t rtp_payload_type) const; + + // Returns kOK if all packets in `packet_list` carry payload types that are + // registered in the database. Otherwise, returns kDecoderNotFound. + int CheckPayloadTypes(const PacketList& packet_list) const; + + private: + typedef std::map DecoderMap; + + DecoderMap decoders_; + int active_decoder_type_; + int active_cng_decoder_type_; + mutable std::unique_ptr active_cng_decoder_; + rtc::scoped_refptr decoder_factory_; + const absl::optional codec_pair_id_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DECODER_DATABASE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database_unittest.cc new file mode 100644 index 0000000000..445c21924b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database_unittest.cc @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/decoder_database.h" + +#include + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder.h" +#include "test/mock_audio_decoder_factory.h" + +using ::testing::_; +using ::testing::Invoke; + +namespace webrtc { + +TEST(DecoderDatabase, CreateAndDestroy) { + DecoderDatabase db(rtc::make_ref_counted(), + absl::nullopt); + EXPECT_EQ(0, db.Size()); + EXPECT_TRUE(db.Empty()); +} + +TEST(DecoderDatabase, InsertAndRemove) { + auto factory = rtc::make_ref_counted(); + DecoderDatabase db(factory, absl::nullopt); + const uint8_t kPayloadType = 0; + const std::string kCodecName = "Robert\'); DROP TABLE Students;"; + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadType, SdpAudioFormat(kCodecName, 8000, 1))); + EXPECT_EQ(1, db.Size()); + EXPECT_FALSE(db.Empty()); + EXPECT_EQ(DecoderDatabase::kOK, db.Remove(kPayloadType)); + EXPECT_EQ(0, db.Size()); + EXPECT_TRUE(db.Empty()); +} + +TEST(DecoderDatabase, InsertAndRemoveAll) { + auto factory = rtc::make_ref_counted(); + DecoderDatabase db(factory, absl::nullopt); + const std::string kCodecName1 = "Robert\'); DROP TABLE Students;"; + const std::string kCodecName2 = "https://xkcd.com/327/"; + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(0, SdpAudioFormat(kCodecName1, 8000, 1))); + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(1, SdpAudioFormat(kCodecName2, 8000, 1))); + EXPECT_EQ(2, db.Size()); + EXPECT_FALSE(db.Empty()); + db.RemoveAll(); + EXPECT_EQ(0, db.Size()); + EXPECT_TRUE(db.Empty()); +} + +TEST(DecoderDatabase, GetDecoderInfo) { + auto factory = rtc::make_ref_counted(); + auto* decoder = new MockAudioDecoder; + EXPECT_CALL(*factory, MakeAudioDecoderMock(_, _, _)) + .WillOnce(Invoke([decoder](const SdpAudioFormat& format, + absl::optional codec_pair_id, + std::unique_ptr* dec) { + EXPECT_EQ("pcmu", format.name); + dec->reset(decoder); + })); + DecoderDatabase db(factory, absl::nullopt); + const uint8_t kPayloadType = 0; + const std::string kCodecName = "pcmu"; + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadType, SdpAudioFormat(kCodecName, 8000, 1))); + const DecoderDatabase::DecoderInfo* info; + info = db.GetDecoderInfo(kPayloadType); + ASSERT_TRUE(info != NULL); + EXPECT_TRUE(info->IsType("pcmu")); + EXPECT_EQ(kCodecName, info->get_name()); + EXPECT_EQ(decoder, db.GetDecoder(kPayloadType)); + info = db.GetDecoderInfo(kPayloadType + 1); // Other payload type. + EXPECT_TRUE(info == NULL); // Should not be found. +} + +TEST(DecoderDatabase, GetDecoder) { + DecoderDatabase db(CreateBuiltinAudioDecoderFactory(), absl::nullopt); + const uint8_t kPayloadType = 0; + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(kPayloadType, SdpAudioFormat("l16", 8000, 1))); + AudioDecoder* dec = db.GetDecoder(kPayloadType); + ASSERT_TRUE(dec != NULL); +} + +TEST(DecoderDatabase, TypeTests) { + auto factory = rtc::make_ref_counted(); + DecoderDatabase db(factory, absl::nullopt); + const uint8_t kPayloadTypePcmU = 0; + const uint8_t kPayloadTypeCng = 13; + const uint8_t kPayloadTypeDtmf = 100; + const uint8_t kPayloadTypeRed = 101; + const uint8_t kPayloadNotUsed = 102; + // Load into database. + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadTypePcmU, SdpAudioFormat("pcmu", 8000, 1))); + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(kPayloadTypeCng, SdpAudioFormat("cn", 8000, 1))); + EXPECT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(kPayloadTypeDtmf, + SdpAudioFormat("telephone-event", 8000, 1))); + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(kPayloadTypeRed, SdpAudioFormat("red", 8000, 1))); + EXPECT_EQ(4, db.Size()); + // Test. + EXPECT_FALSE(db.IsComfortNoise(kPayloadNotUsed)); + EXPECT_FALSE(db.IsDtmf(kPayloadNotUsed)); + EXPECT_FALSE(db.IsRed(kPayloadNotUsed)); + EXPECT_FALSE(db.IsComfortNoise(kPayloadTypePcmU)); + EXPECT_FALSE(db.IsDtmf(kPayloadTypePcmU)); + EXPECT_FALSE(db.IsRed(kPayloadTypePcmU)); + EXPECT_TRUE(db.IsComfortNoise(kPayloadTypeCng)); + EXPECT_TRUE(db.IsDtmf(kPayloadTypeDtmf)); + EXPECT_TRUE(db.IsRed(kPayloadTypeRed)); +} + +TEST(DecoderDatabase, CheckPayloadTypes) { + constexpr int kNumPayloads = 10; + auto factory = rtc::make_ref_counted(); + DecoderDatabase db(factory, absl::nullopt); + // Load a number of payloads into the database. Payload types are 0, 1, ..., + // while the decoder type is the same for all payload types (this does not + // matter for the test). + for (uint8_t payload_type = 0; payload_type < kNumPayloads; ++payload_type) { + EXPECT_EQ( + DecoderDatabase::kOK, + db.RegisterPayload(payload_type, SdpAudioFormat("pcmu", 8000, 1))); + } + PacketList packet_list; + for (int i = 0; i < kNumPayloads + 1; ++i) { + // Create packet with payload type `i`. The last packet will have a payload + // type that is not registered in the decoder database. + Packet packet; + packet.payload_type = i; + packet_list.push_back(std::move(packet)); + } + + // Expect to return false, since the last packet is of an unknown type. + EXPECT_EQ(DecoderDatabase::kDecoderNotFound, + db.CheckPayloadTypes(packet_list)); + + packet_list.pop_back(); // Remove the unknown one. + + EXPECT_EQ(DecoderDatabase::kOK, db.CheckPayloadTypes(packet_list)); + + // Delete all packets. + PacketList::iterator it = packet_list.begin(); + while (it != packet_list.end()) { + it = packet_list.erase(it); + } +} + +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +#define IF_ISAC(x) x +#else +#define IF_ISAC(x) DISABLED_##x +#endif + +// Test the methods for setting and getting active speech and CNG decoders. +TEST(DecoderDatabase, IF_ISAC(ActiveDecoders)) { + DecoderDatabase db(CreateBuiltinAudioDecoderFactory(), absl::nullopt); + // Load payload types. + ASSERT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(0, SdpAudioFormat("pcmu", 8000, 1))); + ASSERT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(103, SdpAudioFormat("isac", 16000, 1))); + ASSERT_EQ(DecoderDatabase::kOK, + db.RegisterPayload(13, SdpAudioFormat("cn", 8000, 1))); + // Verify that no decoders are active from the start. + EXPECT_EQ(NULL, db.GetActiveDecoder()); + EXPECT_EQ(NULL, db.GetActiveCngDecoder()); + + // Set active speech codec. + bool changed; // Should be true when the active decoder changed. + EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveDecoder(0, &changed)); + EXPECT_TRUE(changed); + AudioDecoder* decoder = db.GetActiveDecoder(); + ASSERT_FALSE(decoder == NULL); // Should get a decoder here. + + // Set the same again. Expect no change. + EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveDecoder(0, &changed)); + EXPECT_FALSE(changed); + decoder = db.GetActiveDecoder(); + ASSERT_FALSE(decoder == NULL); // Should get a decoder here. + + // Change active decoder. + EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveDecoder(103, &changed)); + EXPECT_TRUE(changed); + decoder = db.GetActiveDecoder(); + ASSERT_FALSE(decoder == NULL); // Should get a decoder here. + + // Remove the active decoder, and verify that the active becomes NULL. + EXPECT_EQ(DecoderDatabase::kOK, db.Remove(103)); + EXPECT_EQ(NULL, db.GetActiveDecoder()); + + // Set active CNG codec. + EXPECT_EQ(DecoderDatabase::kOK, db.SetActiveCngDecoder(13)); + ComfortNoiseDecoder* cng = db.GetActiveCngDecoder(); + ASSERT_FALSE(cng == NULL); // Should get a decoder here. + + // Remove the active CNG decoder, and verify that the active becomes NULL. + EXPECT_EQ(DecoderDatabase::kOK, db.Remove(13)); + EXPECT_EQ(NULL, db.GetActiveCngDecoder()); + + // Try to set non-existing codecs as active. + EXPECT_EQ(DecoderDatabase::kDecoderNotFound, + db.SetActiveDecoder(17, &changed)); + EXPECT_EQ(DecoderDatabase::kDecoderNotFound, db.SetActiveCngDecoder(17)); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.cc b/third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.cc new file mode 100644 index 0000000000..487450fe0f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.cc @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/default_neteq_factory.h" + +#include + +#include "modules/audio_coding/neteq/neteq_impl.h" + +namespace webrtc { + +DefaultNetEqFactory::DefaultNetEqFactory() = default; +DefaultNetEqFactory::~DefaultNetEqFactory() = default; + +std::unique_ptr DefaultNetEqFactory::CreateNetEq( + const NetEq::Config& config, + const rtc::scoped_refptr& decoder_factory, + Clock* clock) const { + return std::make_unique( + config, NetEqImpl::Dependencies(config, clock, decoder_factory, + controller_factory_)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.h b/third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.h new file mode 100644 index 0000000000..24d2bae419 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/default_neteq_factory.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DEFAULT_NETEQ_FACTORY_H_ +#define MODULES_AUDIO_CODING_NETEQ_DEFAULT_NETEQ_FACTORY_H_ + +#include + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/neteq/default_neteq_controller_factory.h" +#include "api/neteq/neteq_factory.h" +#include "api/scoped_refptr.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +class DefaultNetEqFactory : public NetEqFactory { + public: + DefaultNetEqFactory(); + ~DefaultNetEqFactory() override; + DefaultNetEqFactory(const DefaultNetEqFactory&) = delete; + DefaultNetEqFactory& operator=(const DefaultNetEqFactory&) = delete; + + std::unique_ptr CreateNetEq( + const NetEq::Config& config, + const rtc::scoped_refptr& decoder_factory, + Clock* clock) const override; + + private: + const DefaultNetEqControllerFactory controller_factory_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DEFAULT_NETEQ_FACTORY_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.cc b/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.cc new file mode 100644 index 0000000000..bf3a0f18a1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.cc @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/delay_manager.h" + +#include +#include + +#include +#include +#include +#include + +#include "modules/include/module_common_types_public.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/struct_parameters_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace { + +constexpr int kMinBaseMinimumDelayMs = 0; +constexpr int kMaxBaseMinimumDelayMs = 10000; +constexpr int kStartDelayMs = 80; + +std::unique_ptr MaybeCreateReorderOptimizer( + const DelayManager::Config& config) { + if (!config.use_reorder_optimizer) { + return nullptr; + } + return std::make_unique( + (1 << 15) * config.reorder_forget_factor, config.ms_per_loss_percent, + config.start_forget_weight); +} + +} // namespace + +DelayManager::Config::Config() { + StructParametersParser::Create( // + "quantile", &quantile, // + "forget_factor", &forget_factor, // + "start_forget_weight", &start_forget_weight, // + "resample_interval_ms", &resample_interval_ms, // + "use_reorder_optimizer", &use_reorder_optimizer, // + "reorder_forget_factor", &reorder_forget_factor, // + "ms_per_loss_percent", &ms_per_loss_percent) + ->Parse(webrtc::field_trial::FindFullName( + "WebRTC-Audio-NetEqDelayManagerConfig")); +} + +void DelayManager::Config::Log() { + RTC_LOG(LS_INFO) << "Delay manager config:" + " quantile=" + << quantile << " forget_factor=" << forget_factor + << " start_forget_weight=" << start_forget_weight.value_or(0) + << " resample_interval_ms=" + << resample_interval_ms.value_or(0) + << " use_reorder_optimizer=" << use_reorder_optimizer + << " reorder_forget_factor=" << reorder_forget_factor + << " ms_per_loss_percent=" << ms_per_loss_percent; +} + +DelayManager::DelayManager(const Config& config, const TickTimer* tick_timer) + : max_packets_in_buffer_(config.max_packets_in_buffer), + underrun_optimizer_(tick_timer, + (1 << 30) * config.quantile, + (1 << 15) * config.forget_factor, + config.start_forget_weight, + config.resample_interval_ms), + reorder_optimizer_(MaybeCreateReorderOptimizer(config)), + base_minimum_delay_ms_(config.base_minimum_delay_ms), + effective_minimum_delay_ms_(config.base_minimum_delay_ms), + minimum_delay_ms_(0), + maximum_delay_ms_(0), + target_level_ms_(kStartDelayMs) { + RTC_DCHECK_GE(base_minimum_delay_ms_, 0); + + Reset(); +} + +DelayManager::~DelayManager() {} + +void DelayManager::Update(int arrival_delay_ms, bool reordered) { + if (!reorder_optimizer_ || !reordered) { + underrun_optimizer_.Update(arrival_delay_ms); + } + target_level_ms_ = + underrun_optimizer_.GetOptimalDelayMs().value_or(kStartDelayMs); + if (reorder_optimizer_) { + reorder_optimizer_->Update(arrival_delay_ms, reordered, target_level_ms_); + target_level_ms_ = std::max( + target_level_ms_, reorder_optimizer_->GetOptimalDelayMs().value_or(0)); + } + unlimited_target_level_ms_ = target_level_ms_; + target_level_ms_ = std::max(target_level_ms_, effective_minimum_delay_ms_); + if (maximum_delay_ms_ > 0) { + target_level_ms_ = std::min(target_level_ms_, maximum_delay_ms_); + } + if (packet_len_ms_ > 0) { + // Limit to 75% of maximum buffer size. + target_level_ms_ = std::min( + target_level_ms_, 3 * max_packets_in_buffer_ * packet_len_ms_ / 4); + } +} + +int DelayManager::SetPacketAudioLength(int length_ms) { + if (length_ms <= 0) { + RTC_LOG_F(LS_ERROR) << "length_ms = " << length_ms; + return -1; + } + packet_len_ms_ = length_ms; + return 0; +} + +void DelayManager::Reset() { + packet_len_ms_ = 0; + underrun_optimizer_.Reset(); + target_level_ms_ = kStartDelayMs; + if (reorder_optimizer_) { + reorder_optimizer_->Reset(); + } +} + +int DelayManager::TargetDelayMs() const { + return target_level_ms_; +} + +int DelayManager::UnlimitedTargetLevelMs() const { + return unlimited_target_level_ms_; +} + +bool DelayManager::IsValidMinimumDelay(int delay_ms) const { + return 0 <= delay_ms && delay_ms <= MinimumDelayUpperBound(); +} + +bool DelayManager::IsValidBaseMinimumDelay(int delay_ms) const { + return kMinBaseMinimumDelayMs <= delay_ms && + delay_ms <= kMaxBaseMinimumDelayMs; +} + +bool DelayManager::SetMinimumDelay(int delay_ms) { + if (!IsValidMinimumDelay(delay_ms)) { + return false; + } + + minimum_delay_ms_ = delay_ms; + UpdateEffectiveMinimumDelay(); + return true; +} + +bool DelayManager::SetMaximumDelay(int delay_ms) { + // If `delay_ms` is zero then it unsets the maximum delay and target level is + // unconstrained by maximum delay. + if (delay_ms != 0 && delay_ms < minimum_delay_ms_) { + // Maximum delay shouldn't be less than minimum delay or less than a packet. + return false; + } + + maximum_delay_ms_ = delay_ms; + UpdateEffectiveMinimumDelay(); + return true; +} + +bool DelayManager::SetBaseMinimumDelay(int delay_ms) { + if (!IsValidBaseMinimumDelay(delay_ms)) { + return false; + } + + base_minimum_delay_ms_ = delay_ms; + UpdateEffectiveMinimumDelay(); + return true; +} + +int DelayManager::GetBaseMinimumDelay() const { + return base_minimum_delay_ms_; +} + +void DelayManager::UpdateEffectiveMinimumDelay() { + // Clamp `base_minimum_delay_ms_` into the range which can be effectively + // used. + const int base_minimum_delay_ms = + rtc::SafeClamp(base_minimum_delay_ms_, 0, MinimumDelayUpperBound()); + effective_minimum_delay_ms_ = + std::max(minimum_delay_ms_, base_minimum_delay_ms); +} + +int DelayManager::MinimumDelayUpperBound() const { + // Choose the lowest possible bound discarding 0 cases which mean the value + // is not set and unconstrained. + int q75 = max_packets_in_buffer_ * packet_len_ms_ * 3 / 4; + q75 = q75 > 0 ? q75 : kMaxBaseMinimumDelayMs; + const int maximum_delay_ms = + maximum_delay_ms_ > 0 ? maximum_delay_ms_ : kMaxBaseMinimumDelayMs; + return std::min(maximum_delay_ms, q75); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.h b/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.h new file mode 100644 index 0000000000..a333681535 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DELAY_MANAGER_H_ +#define MODULES_AUDIO_CODING_NETEQ_DELAY_MANAGER_H_ + +#include // Provide access to size_t. + +#include +#include + +#include "absl/types/optional.h" +#include "api/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/histogram.h" +#include "modules/audio_coding/neteq/reorder_optimizer.h" +#include "modules/audio_coding/neteq/underrun_optimizer.h" + +namespace webrtc { + +class DelayManager { + public: + struct Config { + Config(); + void Log(); + + // Options that can be configured via field trial. + double quantile = 0.95; + double forget_factor = 0.983; + absl::optional start_forget_weight = 2; + absl::optional resample_interval_ms = 500; + + bool use_reorder_optimizer = true; + double reorder_forget_factor = 0.9993; + int ms_per_loss_percent = 20; + + // Options that are externally populated. + int max_packets_in_buffer = 200; + int base_minimum_delay_ms = 0; + }; + + DelayManager(const Config& config, const TickTimer* tick_timer); + + virtual ~DelayManager(); + + DelayManager(const DelayManager&) = delete; + DelayManager& operator=(const DelayManager&) = delete; + + // Updates the delay manager that a new packet arrived with delay + // `arrival_delay_ms`. This updates the statistics and a new target buffer + // level is calculated. The `reordered` flag indicates if the packet was + // reordered. + virtual void Update(int arrival_delay_ms, bool reordered); + + // Resets all state. + virtual void Reset(); + + // Gets the target buffer level in milliseconds. If a minimum or maximum delay + // has been set, the target delay reported here also respects the configured + // min/max delay. + virtual int TargetDelayMs() const; + + // Reports the target delay that would be used if no minimum/maximum delay + // would be set. + virtual int UnlimitedTargetLevelMs() const; + + // Notifies the DelayManager of how much audio data is carried in each packet. + virtual int SetPacketAudioLength(int length_ms); + + // Accessors and mutators. + // Assuming `delay` is in valid range. + virtual bool SetMinimumDelay(int delay_ms); + virtual bool SetMaximumDelay(int delay_ms); + virtual bool SetBaseMinimumDelay(int delay_ms); + virtual int GetBaseMinimumDelay() const; + + // These accessors are only intended for testing purposes. + int effective_minimum_delay_ms_for_test() const { + return effective_minimum_delay_ms_; + } + + private: + // Provides value which minimum delay can't exceed based on current buffer + // size and given `maximum_delay_ms_`. Lower bound is a constant 0. + int MinimumDelayUpperBound() const; + + // Updates `effective_minimum_delay_ms_` delay based on current + // `minimum_delay_ms_`, `base_minimum_delay_ms_` and `maximum_delay_ms_` + // and buffer size. + void UpdateEffectiveMinimumDelay(); + + // Makes sure that `delay_ms` is less than maximum delay, if any maximum + // is set. Also, if possible check `delay_ms` to be less than 75% of + // `max_packets_in_buffer_`. + bool IsValidMinimumDelay(int delay_ms) const; + + bool IsValidBaseMinimumDelay(int delay_ms) const; + + // TODO(jakobi): set maximum buffer delay instead of number of packets. + const int max_packets_in_buffer_; + UnderrunOptimizer underrun_optimizer_; + std::unique_ptr reorder_optimizer_; + + int base_minimum_delay_ms_; + int effective_minimum_delay_ms_; // Used as lower bound for target delay. + int minimum_delay_ms_; // Externally set minimum delay. + int maximum_delay_ms_; // Externally set maximum allowed delay. + + int packet_len_ms_ = 0; + int target_level_ms_ = 0; // Currently preferred buffer level. + int unlimited_target_level_ms_ = 0; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DELAY_MANAGER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager_unittest.cc new file mode 100644 index 0000000000..da5f53188c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager_unittest.cc @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for DelayManager class. + +#include "modules/audio_coding/neteq/delay_manager.h" + +#include + +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/histogram.h" +#include "modules/audio_coding/neteq/mock/mock_histogram.h" +#include "modules/audio_coding/neteq/mock/mock_statistics_calculator.h" +#include "rtc_base/checks.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr int kMaxNumberOfPackets = 200; +constexpr int kTimeStepMs = 10; +constexpr int kFrameSizeMs = 20; +constexpr int kMaxBufferSizeMs = kMaxNumberOfPackets * kFrameSizeMs; + +} // namespace + +class DelayManagerTest : public ::testing::Test { + protected: + DelayManagerTest(); + virtual void SetUp(); + void Update(int delay); + void IncreaseTime(int inc_ms); + + TickTimer tick_timer_; + DelayManager dm_; +}; + +DelayManagerTest::DelayManagerTest() + : dm_(DelayManager::Config(), &tick_timer_) {} + +void DelayManagerTest::SetUp() { + dm_.SetPacketAudioLength(kFrameSizeMs); +} + +void DelayManagerTest::Update(int delay) { + dm_.Update(delay, false); +} + +void DelayManagerTest::IncreaseTime(int inc_ms) { + for (int t = 0; t < inc_ms; t += kTimeStepMs) { + tick_timer_.Increment(); + } +} + +TEST_F(DelayManagerTest, CreateAndDestroy) { + // Nothing to do here. The test fixture creates and destroys the DelayManager + // object. +} + +TEST_F(DelayManagerTest, UpdateNormal) { + for (int i = 0; i < 50; ++i) { + Update(0); + IncreaseTime(kFrameSizeMs); + } + EXPECT_EQ(20, dm_.TargetDelayMs()); +} + +TEST_F(DelayManagerTest, MaxDelay) { + Update(0); + const int kMaxDelayMs = 60; + EXPECT_GT(dm_.TargetDelayMs(), kMaxDelayMs); + EXPECT_TRUE(dm_.SetMaximumDelay(kMaxDelayMs)); + Update(0); + EXPECT_EQ(kMaxDelayMs, dm_.TargetDelayMs()); +} + +TEST_F(DelayManagerTest, MinDelay) { + Update(0); + int kMinDelayMs = 7 * kFrameSizeMs; + EXPECT_LT(dm_.TargetDelayMs(), kMinDelayMs); + dm_.SetMinimumDelay(kMinDelayMs); + IncreaseTime(kFrameSizeMs); + Update(0); + EXPECT_EQ(kMinDelayMs, dm_.TargetDelayMs()); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayCheckValidRange) { + // Base minimum delay should be between [0, 10000] milliseconds. + EXPECT_FALSE(dm_.SetBaseMinimumDelay(-1)); + EXPECT_FALSE(dm_.SetBaseMinimumDelay(10001)); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), 0); + + EXPECT_TRUE(dm_.SetBaseMinimumDelay(7999)); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), 7999); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayLowerThanMinimumDelay) { + constexpr int kBaseMinimumDelayMs = 100; + constexpr int kMinimumDelayMs = 200; + + // Base minimum delay sets lower bound on minimum. That is why when base + // minimum delay is lower than minimum delay we use minimum delay. + RTC_DCHECK_LT(kBaseMinimumDelayMs, kMinimumDelayMs); + + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMinimumDelayMs); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanMinimumDelay) { + constexpr int kBaseMinimumDelayMs = 70; + constexpr int kMinimumDelayMs = 30; + + // Base minimum delay sets lower bound on minimum. That is why when base + // minimum delay is greater than minimum delay we use base minimum delay. + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); + + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMs); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanBufferSize) { + constexpr int kBaseMinimumDelayMs = kMaxBufferSizeMs + 1; + constexpr int kMinimumDelayMs = 12; + constexpr int kMaximumDelayMs = 20; + constexpr int kMaxBufferSizeMsQ75 = 3 * kMaxBufferSizeMs / 4; + + EXPECT_TRUE(dm_.SetMaximumDelay(kMaximumDelayMs)); + + // Base minimum delay is greater than minimum delay, that is why we clamp + // it to current the highest possible value which is maximum delay. + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMaxBufferSizeMs); + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMaximumDelayMs); + RTC_DCHECK_LT(kMaximumDelayMs, kMaxBufferSizeMsQ75); + + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + + // Unset maximum value. + EXPECT_TRUE(dm_.SetMaximumDelay(0)); + + // With maximum value unset, the highest possible value now is 75% of + // currently possible maximum buffer size. + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMaxBufferSizeMsQ75); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanMaximumDelay) { + constexpr int kMaximumDelayMs = 400; + constexpr int kBaseMinimumDelayMs = kMaximumDelayMs + 1; + constexpr int kMinimumDelayMs = 20; + + // Base minimum delay is greater than minimum delay, that is why we clamp + // it to current the highest possible value which is kMaximumDelayMs. + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMaximumDelayMs); + RTC_DCHECK_LT(kMaximumDelayMs, kMaxBufferSizeMs); + + EXPECT_TRUE(dm_.SetMaximumDelay(kMaximumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMaximumDelayMs); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayLowerThanMaxSize) { + constexpr int kMaximumDelayMs = 400; + constexpr int kBaseMinimumDelayMs = kMaximumDelayMs - 1; + constexpr int kMinimumDelayMs = 20; + + // Base minimum delay is greater than minimum delay, and lower than maximum + // delays that is why it is used. + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); + RTC_DCHECK_LT(kBaseMinimumDelayMs, kMaximumDelayMs); + + EXPECT_TRUE(dm_.SetMaximumDelay(kMaximumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMs); +} + +TEST_F(DelayManagerTest, MinimumDelayMemorization) { + // Check that when we increase base minimum delay to value higher than + // minimum delay then minimum delay is still memorized. This allows to + // restore effective minimum delay to memorized minimum delay value when we + // decrease base minimum delay. + constexpr int kBaseMinimumDelayMsLow = 10; + constexpr int kMinimumDelayMs = 20; + constexpr int kBaseMinimumDelayMsHigh = 30; + + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMsLow)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + // Minimum delay is used as it is higher than base minimum delay. + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMinimumDelayMs); + + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMsHigh)); + // Base minimum delay is used as it is now higher than minimum delay. + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMsHigh); + + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMsLow)); + // Check that minimum delay is memorized and is used again. + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMinimumDelayMs); +} + +TEST_F(DelayManagerTest, BaseMinimumDelay) { + // First packet arrival. + Update(0); + + constexpr int kBaseMinimumDelayMs = 7 * kFrameSizeMs; + EXPECT_LT(dm_.TargetDelayMs(), kBaseMinimumDelayMs); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), kBaseMinimumDelayMs); + + IncreaseTime(kFrameSizeMs); + Update(0); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), kBaseMinimumDelayMs); + EXPECT_EQ(kBaseMinimumDelayMs, dm_.TargetDelayMs()); +} + +TEST_F(DelayManagerTest, Failures) { + // Wrong packet size. + EXPECT_EQ(-1, dm_.SetPacketAudioLength(0)); + EXPECT_EQ(-1, dm_.SetPacketAudioLength(-1)); + + // Minimum delay higher than a maximum delay is not accepted. + EXPECT_TRUE(dm_.SetMaximumDelay(20)); + EXPECT_FALSE(dm_.SetMinimumDelay(40)); + + // Maximum delay less than minimum delay is not accepted. + EXPECT_TRUE(dm_.SetMaximumDelay(100)); + EXPECT_TRUE(dm_.SetMinimumDelay(80)); + EXPECT_FALSE(dm_.SetMaximumDelay(60)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.cc b/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.cc new file mode 100644 index 0000000000..a979f94214 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.cc @@ -0,0 +1,373 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/dsp_helper.h" + +#include // Access to memset. + +#include // Access to min, max. + +#include "common_audio/signal_processing/include/signal_processing_library.h" + +namespace webrtc { + +// Table of constants used in method DspHelper::ParabolicFit(). +const int16_t DspHelper::kParabolaCoefficients[17][3] = { + {120, 32, 64}, {140, 44, 75}, {150, 50, 80}, {160, 57, 85}, + {180, 72, 96}, {200, 89, 107}, {210, 98, 112}, {220, 108, 117}, + {240, 128, 128}, {260, 150, 139}, {270, 162, 144}, {280, 174, 149}, + {300, 200, 160}, {320, 228, 171}, {330, 242, 176}, {340, 257, 181}, + {360, 288, 192}}; + +// Filter coefficients used when downsampling from the indicated sample rates +// (8, 16, 32, 48 kHz) to 4 kHz. Coefficients are in Q12. The corresponding Q0 +// values are provided in the comments before each array. + +// Q0 values: {0.3, 0.4, 0.3}. +const int16_t DspHelper::kDownsample8kHzTbl[3] = {1229, 1638, 1229}; + +// Q0 values: {0.15, 0.2, 0.3, 0.2, 0.15}. +const int16_t DspHelper::kDownsample16kHzTbl[5] = {614, 819, 1229, 819, 614}; + +// Q0 values: {0.1425, 0.1251, 0.1525, 0.1628, 0.1525, 0.1251, 0.1425}. +const int16_t DspHelper::kDownsample32kHzTbl[7] = {584, 512, 625, 667, + 625, 512, 584}; + +// Q0 values: {0.2487, 0.0952, 0.1042, 0.1074, 0.1042, 0.0952, 0.2487}. +const int16_t DspHelper::kDownsample48kHzTbl[7] = {1019, 390, 427, 440, + 427, 390, 1019}; + +int DspHelper::RampSignal(const int16_t* input, + size_t length, + int factor, + int increment, + int16_t* output) { + int factor_q20 = (factor << 6) + 32; + // TODO(hlundin): Add 32 to factor_q20 when converting back to Q14? + for (size_t i = 0; i < length; ++i) { + output[i] = (factor * input[i] + 8192) >> 14; + factor_q20 += increment; + factor_q20 = std::max(factor_q20, 0); // Never go negative. + factor = std::min(factor_q20 >> 6, 16384); + } + return factor; +} + +int DspHelper::RampSignal(int16_t* signal, + size_t length, + int factor, + int increment) { + return RampSignal(signal, length, factor, increment, signal); +} + +int DspHelper::RampSignal(AudioVector* signal, + size_t start_index, + size_t length, + int factor, + int increment) { + int factor_q20 = (factor << 6) + 32; + // TODO(hlundin): Add 32 to factor_q20 when converting back to Q14? + for (size_t i = start_index; i < start_index + length; ++i) { + (*signal)[i] = (factor * (*signal)[i] + 8192) >> 14; + factor_q20 += increment; + factor_q20 = std::max(factor_q20, 0); // Never go negative. + factor = std::min(factor_q20 >> 6, 16384); + } + return factor; +} + +int DspHelper::RampSignal(AudioMultiVector* signal, + size_t start_index, + size_t length, + int factor, + int increment) { + RTC_DCHECK_LE(start_index + length, signal->Size()); + if (start_index + length > signal->Size()) { + // Wrong parameters. Do nothing and return the scale factor unaltered. + return factor; + } + int end_factor = 0; + // Loop over the channels, starting at the same `factor` each time. + for (size_t channel = 0; channel < signal->Channels(); ++channel) { + end_factor = + RampSignal(&(*signal)[channel], start_index, length, factor, increment); + } + return end_factor; +} + +void DspHelper::PeakDetection(int16_t* data, + size_t data_length, + size_t num_peaks, + int fs_mult, + size_t* peak_index, + int16_t* peak_value) { + size_t min_index = 0; + size_t max_index = 0; + + for (size_t i = 0; i <= num_peaks - 1; i++) { + if (num_peaks == 1) { + // Single peak. The parabola fit assumes that an extra point is + // available; worst case it gets a zero on the high end of the signal. + // TODO(hlundin): This can potentially get much worse. It breaks the + // API contract, that the length of `data` is `data_length`. + data_length++; + } + + peak_index[i] = WebRtcSpl_MaxIndexW16(data, data_length - 1); + + if (i != num_peaks - 1) { + min_index = (peak_index[i] > 2) ? (peak_index[i] - 2) : 0; + max_index = std::min(data_length - 1, peak_index[i] + 2); + } + + if ((peak_index[i] != 0) && (peak_index[i] != (data_length - 2))) { + ParabolicFit(&data[peak_index[i] - 1], fs_mult, &peak_index[i], + &peak_value[i]); + } else { + if (peak_index[i] == data_length - 2) { + if (data[peak_index[i]] > data[peak_index[i] + 1]) { + ParabolicFit(&data[peak_index[i] - 1], fs_mult, &peak_index[i], + &peak_value[i]); + } else if (data[peak_index[i]] <= data[peak_index[i] + 1]) { + // Linear approximation. + peak_value[i] = (data[peak_index[i]] + data[peak_index[i] + 1]) >> 1; + peak_index[i] = (peak_index[i] * 2 + 1) * fs_mult; + } + } else { + peak_value[i] = data[peak_index[i]]; + peak_index[i] = peak_index[i] * 2 * fs_mult; + } + } + + if (i != num_peaks - 1) { + memset(&data[min_index], 0, + sizeof(data[0]) * (max_index - min_index + 1)); + } + } +} + +void DspHelper::ParabolicFit(int16_t* signal_points, + int fs_mult, + size_t* peak_index, + int16_t* peak_value) { + uint16_t fit_index[13]; + if (fs_mult == 1) { + fit_index[0] = 0; + fit_index[1] = 8; + fit_index[2] = 16; + } else if (fs_mult == 2) { + fit_index[0] = 0; + fit_index[1] = 4; + fit_index[2] = 8; + fit_index[3] = 12; + fit_index[4] = 16; + } else if (fs_mult == 4) { + fit_index[0] = 0; + fit_index[1] = 2; + fit_index[2] = 4; + fit_index[3] = 6; + fit_index[4] = 8; + fit_index[5] = 10; + fit_index[6] = 12; + fit_index[7] = 14; + fit_index[8] = 16; + } else { + fit_index[0] = 0; + fit_index[1] = 1; + fit_index[2] = 3; + fit_index[3] = 4; + fit_index[4] = 5; + fit_index[5] = 7; + fit_index[6] = 8; + fit_index[7] = 9; + fit_index[8] = 11; + fit_index[9] = 12; + fit_index[10] = 13; + fit_index[11] = 15; + fit_index[12] = 16; + } + + // num = -3 * signal_points[0] + 4 * signal_points[1] - signal_points[2]; + // den = signal_points[0] - 2 * signal_points[1] + signal_points[2]; + int32_t num = + (signal_points[0] * -3) + (signal_points[1] * 4) - signal_points[2]; + int32_t den = signal_points[0] + (signal_points[1] * -2) + signal_points[2]; + int32_t temp = num * 120; + int flag = 1; + int16_t stp = kParabolaCoefficients[fit_index[fs_mult]][0] - + kParabolaCoefficients[fit_index[fs_mult - 1]][0]; + int16_t strt = (kParabolaCoefficients[fit_index[fs_mult]][0] + + kParabolaCoefficients[fit_index[fs_mult - 1]][0]) / + 2; + int16_t lmt; + if (temp < -den * strt) { + lmt = strt - stp; + while (flag) { + if ((flag == fs_mult) || (temp > -den * lmt)) { + *peak_value = + (den * kParabolaCoefficients[fit_index[fs_mult - flag]][1] + + num * kParabolaCoefficients[fit_index[fs_mult - flag]][2] + + signal_points[0] * 256) / + 256; + *peak_index = *peak_index * 2 * fs_mult - flag; + flag = 0; + } else { + flag++; + lmt -= stp; + } + } + } else if (temp > -den * (strt + stp)) { + lmt = strt + 2 * stp; + while (flag) { + if ((flag == fs_mult) || (temp < -den * lmt)) { + int32_t temp_term_1 = + den * kParabolaCoefficients[fit_index[fs_mult + flag]][1]; + int32_t temp_term_2 = + num * kParabolaCoefficients[fit_index[fs_mult + flag]][2]; + int32_t temp_term_3 = signal_points[0] * 256; + *peak_value = (temp_term_1 + temp_term_2 + temp_term_3) / 256; + *peak_index = *peak_index * 2 * fs_mult + flag; + flag = 0; + } else { + flag++; + lmt += stp; + } + } + } else { + *peak_value = signal_points[1]; + *peak_index = *peak_index * 2 * fs_mult; + } +} + +size_t DspHelper::MinDistortion(const int16_t* signal, + size_t min_lag, + size_t max_lag, + size_t length, + int32_t* distortion_value) { + size_t best_index = 0; + int32_t min_distortion = WEBRTC_SPL_WORD32_MAX; + for (size_t i = min_lag; i <= max_lag; i++) { + int32_t sum_diff = 0; + const int16_t* data1 = signal; + const int16_t* data2 = signal - i; + for (size_t j = 0; j < length; j++) { + sum_diff += WEBRTC_SPL_ABS_W32(data1[j] - data2[j]); + } + // Compare with previous minimum. + if (sum_diff < min_distortion) { + min_distortion = sum_diff; + best_index = i; + } + } + *distortion_value = min_distortion; + return best_index; +} + +void DspHelper::CrossFade(const int16_t* input1, + const int16_t* input2, + size_t length, + int16_t* mix_factor, + int16_t factor_decrement, + int16_t* output) { + int16_t factor = *mix_factor; + int16_t complement_factor = 16384 - factor; + for (size_t i = 0; i < length; i++) { + output[i] = + (factor * input1[i] + complement_factor * input2[i] + 8192) >> 14; + factor -= factor_decrement; + complement_factor += factor_decrement; + } + *mix_factor = factor; +} + +void DspHelper::UnmuteSignal(const int16_t* input, + size_t length, + int16_t* factor, + int increment, + int16_t* output) { + uint16_t factor_16b = *factor; + int32_t factor_32b = (static_cast(factor_16b) << 6) + 32; + for (size_t i = 0; i < length; i++) { + output[i] = (factor_16b * input[i] + 8192) >> 14; + factor_32b = std::max(factor_32b + increment, 0); + factor_16b = std::min(16384, factor_32b >> 6); + } + *factor = factor_16b; +} + +void DspHelper::MuteSignal(int16_t* signal, int mute_slope, size_t length) { + int32_t factor = (16384 << 6) + 32; + for (size_t i = 0; i < length; i++) { + signal[i] = ((factor >> 6) * signal[i] + 8192) >> 14; + factor -= mute_slope; + } +} + +int DspHelper::DownsampleTo4kHz(const int16_t* input, + size_t input_length, + size_t output_length, + int input_rate_hz, + bool compensate_delay, + int16_t* output) { + // Set filter parameters depending on input frequency. + // NOTE: The phase delay values are wrong compared to the true phase delay + // of the filters. However, the error is preserved (through the +1 term) for + // consistency. + const int16_t* filter_coefficients; // Filter coefficients. + size_t filter_length; // Number of coefficients. + size_t filter_delay; // Phase delay in samples. + int16_t factor; // Conversion rate (inFsHz / 8000). + switch (input_rate_hz) { + case 8000: { + filter_length = 3; + factor = 2; + filter_coefficients = kDownsample8kHzTbl; + filter_delay = 1 + 1; + break; + } + case 16000: { + filter_length = 5; + factor = 4; + filter_coefficients = kDownsample16kHzTbl; + filter_delay = 2 + 1; + break; + } + case 32000: { + filter_length = 7; + factor = 8; + filter_coefficients = kDownsample32kHzTbl; + filter_delay = 3 + 1; + break; + } + case 48000: { + filter_length = 7; + factor = 12; + filter_coefficients = kDownsample48kHzTbl; + filter_delay = 3 + 1; + break; + } + default: { + RTC_DCHECK_NOTREACHED(); + return -1; + } + } + + if (!compensate_delay) { + // Disregard delay compensation. + filter_delay = 0; + } + + // Returns -1 if input signal is too short; 0 otherwise. + return WebRtcSpl_DownsampleFast( + &input[filter_length - 1], input_length - filter_length + 1, output, + output_length, filter_coefficients, filter_length, factor, filter_delay); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.h b/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.h new file mode 100644 index 0000000000..4aead7df18 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DSP_HELPER_H_ +#define MODULES_AUDIO_CODING_NETEQ_DSP_HELPER_H_ + +#include +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/audio_vector.h" + +namespace webrtc { + +// This class contains various signal processing functions, all implemented as +// static methods. +class DspHelper { + public: + // Filter coefficients used when downsampling from the indicated sample rates + // (8, 16, 32, 48 kHz) to 4 kHz. Coefficients are in Q12. + static const int16_t kDownsample8kHzTbl[3]; + static const int16_t kDownsample16kHzTbl[5]; + static const int16_t kDownsample32kHzTbl[7]; + static const int16_t kDownsample48kHzTbl[7]; + + // Constants used to mute and unmute over 5 samples. The coefficients are + // in Q15. + static const int kMuteFactorStart8kHz = 27307; + static const int kMuteFactorIncrement8kHz = -5461; + static const int kUnmuteFactorStart8kHz = 5461; + static const int kUnmuteFactorIncrement8kHz = 5461; + static const int kMuteFactorStart16kHz = 29789; + static const int kMuteFactorIncrement16kHz = -2979; + static const int kUnmuteFactorStart16kHz = 2979; + static const int kUnmuteFactorIncrement16kHz = 2979; + static const int kMuteFactorStart32kHz = 31208; + static const int kMuteFactorIncrement32kHz = -1560; + static const int kUnmuteFactorStart32kHz = 1560; + static const int kUnmuteFactorIncrement32kHz = 1560; + static const int kMuteFactorStart48kHz = 31711; + static const int kMuteFactorIncrement48kHz = -1057; + static const int kUnmuteFactorStart48kHz = 1057; + static const int kUnmuteFactorIncrement48kHz = 1057; + + // Multiplies the signal with a gradually changing factor. + // The first sample is multiplied with `factor` (in Q14). For each sample, + // `factor` is increased (additive) by the `increment` (in Q20), which can + // be negative. Returns the scale factor after the last increment. + static int RampSignal(const int16_t* input, + size_t length, + int factor, + int increment, + int16_t* output); + + // Same as above, but with the samples of `signal` being modified in-place. + static int RampSignal(int16_t* signal, + size_t length, + int factor, + int increment); + + // Same as above, but processes `length` samples from `signal`, starting at + // `start_index`. + static int RampSignal(AudioVector* signal, + size_t start_index, + size_t length, + int factor, + int increment); + + // Same as above, but for an AudioMultiVector. + static int RampSignal(AudioMultiVector* signal, + size_t start_index, + size_t length, + int factor, + int increment); + + // Peak detection with parabolic fit. Looks for `num_peaks` maxima in `data`, + // having length `data_length` and sample rate multiplier `fs_mult`. The peak + // locations and values are written to the arrays `peak_index` and + // `peak_value`, respectively. Both arrays must hold at least `num_peaks` + // elements. + static void PeakDetection(int16_t* data, + size_t data_length, + size_t num_peaks, + int fs_mult, + size_t* peak_index, + int16_t* peak_value); + + // Estimates the height and location of a maximum. The three values in the + // array `signal_points` are used as basis for a parabolic fit, which is then + // used to find the maximum in an interpolated signal. The `signal_points` are + // assumed to be from a 4 kHz signal, while the maximum, written to + // `peak_index` and `peak_value` is given in the full sample rate, as + // indicated by the sample rate multiplier `fs_mult`. + static void ParabolicFit(int16_t* signal_points, + int fs_mult, + size_t* peak_index, + int16_t* peak_value); + + // Calculates the sum-abs-diff for `signal` when compared to a displaced + // version of itself. Returns the displacement lag that results in the minimum + // distortion. The resulting distortion is written to `distortion_value`. + // The values of `min_lag` and `max_lag` are boundaries for the search. + static size_t MinDistortion(const int16_t* signal, + size_t min_lag, + size_t max_lag, + size_t length, + int32_t* distortion_value); + + // Mixes `length` samples from `input1` and `input2` together and writes the + // result to `output`. The gain for `input1` starts at `mix_factor` (Q14) and + // is decreased by `factor_decrement` (Q14) for each sample. The gain for + // `input2` is the complement 16384 - mix_factor. + static void CrossFade(const int16_t* input1, + const int16_t* input2, + size_t length, + int16_t* mix_factor, + int16_t factor_decrement, + int16_t* output); + + // Scales `input` with an increasing gain. Applies `factor` (Q14) to the first + // sample and increases the gain by `increment` (Q20) for each sample. The + // result is written to `output`. `length` samples are processed. + static void UnmuteSignal(const int16_t* input, + size_t length, + int16_t* factor, + int increment, + int16_t* output); + + // Starts at unity gain and gradually fades out `signal`. For each sample, + // the gain is reduced by `mute_slope` (Q14). `length` samples are processed. + static void MuteSignal(int16_t* signal, int mute_slope, size_t length); + + // Downsamples `input` from `sample_rate_hz` to 4 kHz sample rate. The input + // has `input_length` samples, and the method will write `output_length` + // samples to `output`. Compensates for the phase delay of the downsampling + // filters if `compensate_delay` is true. Returns -1 if the input is too short + // to produce `output_length` samples, otherwise 0. + static int DownsampleTo4kHz(const int16_t* input, + size_t input_length, + size_t output_length, + int input_rate_hz, + bool compensate_delay, + int16_t* output); + + DspHelper(const DspHelper&) = delete; + DspHelper& operator=(const DspHelper&) = delete; + + private: + // Table of constants used in method DspHelper::ParabolicFit(). + static const int16_t kParabolaCoefficients[17][3]; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DSP_HELPER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper_unittest.cc new file mode 100644 index 0000000000..09247417d3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper_unittest.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/dsp_helper.h" + +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(DspHelper, RampSignalArray) { + static const int kLen = 100; + int16_t input[kLen]; + int16_t output[kLen]; + // Fill input with 1000. + for (int i = 0; i < kLen; ++i) { + input[i] = 1000; + } + int start_factor = 0; + // Ramp from 0 to 1 (in Q14) over the array. Note that `increment` is in Q20, + // while the factor is in Q14, hence the shift by 6. + int increment = (16384 << 6) / kLen; + + // Test first method. + int stop_factor = + DspHelper::RampSignal(input, kLen, start_factor, increment, output); + EXPECT_EQ(16383, stop_factor); // Almost reach 1 in Q14. + for (int i = 0; i < kLen; ++i) { + EXPECT_EQ(1000 * i / kLen, output[i]); + } + + // Test second method. (Note that this modifies `input`.) + stop_factor = DspHelper::RampSignal(input, kLen, start_factor, increment); + EXPECT_EQ(16383, stop_factor); // Almost reach 1 in Q14. + for (int i = 0; i < kLen; ++i) { + EXPECT_EQ(1000 * i / kLen, input[i]); + } +} + +TEST(DspHelper, RampSignalAudioMultiVector) { + static const int kLen = 100; + static const int kChannels = 5; + AudioMultiVector input(kChannels, kLen * 3); + // Fill input with 1000. + for (int i = 0; i < kLen * 3; ++i) { + for (int channel = 0; channel < kChannels; ++channel) { + input[channel][i] = 1000; + } + } + // We want to start ramping at `start_index` and keep ramping for `kLen` + // samples. + int start_index = kLen; + int start_factor = 0; + // Ramp from 0 to 1 (in Q14) in `kLen` samples. Note that `increment` is in + // Q20, while the factor is in Q14, hence the shift by 6. + int increment = (16384 << 6) / kLen; + + int stop_factor = + DspHelper::RampSignal(&input, start_index, kLen, start_factor, increment); + EXPECT_EQ(16383, stop_factor); // Almost reach 1 in Q14. + // Verify that the first `kLen` samples are left untouched. + int i; + for (i = 0; i < kLen; ++i) { + for (int channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(1000, input[channel][i]); + } + } + // Verify that the next block of `kLen` samples are ramped. + for (; i < 2 * kLen; ++i) { + for (int channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(1000 * (i - kLen) / kLen, input[channel][i]); + } + } + // Verify the last `kLen` samples are left untouched. + for (; i < 3 * kLen; ++i) { + for (int channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(1000, input[channel][i]); + } + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.cc b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.cc new file mode 100644 index 0000000000..115bfcf97b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.cc @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/dtmf_buffer.h" + +#include // max + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +// Modify the code to obtain backwards bit-exactness. Once bit-exactness is no +// longer required, this #define should be removed (and the code that it +// enables). +#define LEGACY_BITEXACT + +namespace webrtc { + +DtmfBuffer::DtmfBuffer(int fs_hz) { + SetSampleRate(fs_hz); +} + +DtmfBuffer::~DtmfBuffer() = default; + +void DtmfBuffer::Flush() { + buffer_.clear(); +} + +// The ParseEvent method parses 4 bytes from `payload` according to this format +// from RFC 4733: +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | event |E|R| volume | duration | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Legend (adapted from RFC 4733) +// - event: The event field is a number between 0 and 255 identifying a +// specific telephony event. The buffer will not accept any event +// numbers larger than 15. +// - E: If set to a value of one, the "end" bit indicates that this +// packet contains the end of the event. For long-lasting events +// that have to be split into segments, only the final packet for +// the final segment will have the E bit set. +// - R: Reserved. +// - volume: For DTMF digits and other events representable as tones, this +// field describes the power level of the tone, expressed in dBm0 +// after dropping the sign. Power levels range from 0 to -63 dBm0. +// Thus, larger values denote lower volume. The buffer discards +// values larger than 36 (i.e., lower than -36 dBm0). +// - duration: The duration field indicates the duration of the event or segment +// being reported, in timestamp units, expressed as an unsigned +// integer in network byte order. For a non-zero value, the event +// or segment began at the instant identified by the RTP timestamp +// and has so far lasted as long as indicated by this parameter. +// The event may or may not have ended. If the event duration +// exceeds the maximum representable by the duration field, the +// event is split into several contiguous segments. The buffer will +// discard zero-duration events. +// +int DtmfBuffer::ParseEvent(uint32_t rtp_timestamp, + const uint8_t* payload, + size_t payload_length_bytes, + DtmfEvent* event) { + RTC_CHECK(payload); + RTC_CHECK(event); + if (payload_length_bytes < 4) { + RTC_LOG(LS_WARNING) << "ParseEvent payload too short"; + return kPayloadTooShort; + } + + event->event_no = payload[0]; + event->end_bit = ((payload[1] & 0x80) != 0); + event->volume = (payload[1] & 0x3F); + event->duration = payload[2] << 8 | payload[3]; + event->timestamp = rtp_timestamp; + return kOK; +} + +// Inserts a DTMF event into the buffer. The event should be parsed from the +// bit stream using the ParseEvent method above before inserting it in the +// buffer. +// DTMF events can be quite long, and in most cases the duration of the event +// is not known when the first packet describing it is sent. To deal with that, +// the RFC 4733 specifies that multiple packets are sent for one and the same +// event as it is being created (typically, as the user is pressing the key). +// These packets will all share the same start timestamp and event number, +// while the duration will be the cumulative duration from the start. When +// inserting a new event, the InsertEvent method tries to find a matching event +// already in the buffer. If so, the new event is simply merged with the +// existing one. +int DtmfBuffer::InsertEvent(const DtmfEvent& event) { + if (event.event_no < 0 || event.event_no > 15 || event.volume < 0 || + event.volume > 63 || event.duration <= 0 || event.duration > 65535) { + RTC_LOG(LS_WARNING) << "InsertEvent invalid parameters"; + return kInvalidEventParameters; + } + DtmfList::iterator it = buffer_.begin(); + while (it != buffer_.end()) { + if (MergeEvents(it, event)) { + // A matching event was found and the new event was merged. + return kOK; + } + ++it; + } + buffer_.push_back(event); + // Sort the buffer using CompareEvents to rank the events. + buffer_.sort(CompareEvents); + return kOK; +} + +bool DtmfBuffer::GetEvent(uint32_t current_timestamp, DtmfEvent* event) { + DtmfList::iterator it = buffer_.begin(); + while (it != buffer_.end()) { + // `event_end` is an estimate of where the current event ends. If the end + // bit is set, we know that the event ends at `timestamp` + `duration`. + uint32_t event_end = it->timestamp + it->duration; +#ifdef LEGACY_BITEXACT + bool next_available = false; +#endif + if (!it->end_bit) { + // If the end bit is not set, we allow extrapolation of the event for + // some time. + event_end += max_extrapolation_samples_; + DtmfList::iterator next = it; + ++next; + if (next != buffer_.end()) { + // If there is a next event in the buffer, we will not extrapolate over + // the start of that new event. + event_end = std::min(event_end, next->timestamp); +#ifdef LEGACY_BITEXACT + next_available = true; +#endif + } + } + if (current_timestamp >= it->timestamp && + current_timestamp <= event_end) { // TODO(hlundin): Change to <. + // Found a matching event. + if (event) { + event->event_no = it->event_no; + event->end_bit = it->end_bit; + event->volume = it->volume; + event->duration = it->duration; + event->timestamp = it->timestamp; + } +#ifdef LEGACY_BITEXACT + if (it->end_bit && current_timestamp + frame_len_samples_ >= event_end) { + // We are done playing this. Erase the event. + buffer_.erase(it); + } +#endif + return true; + } else if (current_timestamp > event_end) { // TODO(hlundin): Change to >=. +// Erase old event. Operation returns a valid pointer to the next element +// in the list. +#ifdef LEGACY_BITEXACT + if (!next_available) { + if (event) { + event->event_no = it->event_no; + event->end_bit = it->end_bit; + event->volume = it->volume; + event->duration = it->duration; + event->timestamp = it->timestamp; + } + it = buffer_.erase(it); + return true; + } else { + it = buffer_.erase(it); + } +#else + it = buffer_.erase(it); +#endif + } else { + ++it; + } + } + return false; +} + +size_t DtmfBuffer::Length() const { + return buffer_.size(); +} + +bool DtmfBuffer::Empty() const { + return buffer_.empty(); +} + +int DtmfBuffer::SetSampleRate(int fs_hz) { + if (fs_hz != 8000 && + fs_hz != 16000 && + fs_hz != 32000 && + fs_hz != 44100 && + fs_hz != 48000) { + return kInvalidSampleRate; + } + max_extrapolation_samples_ = 7 * fs_hz / 100; + frame_len_samples_ = fs_hz / 100; + return kOK; +} + +// The method returns true if the two events are considered to be the same. +// The are defined as equal if they share the same timestamp and event number. +// The special case with long-lasting events that have to be split into segments +// is not handled in this method. These will be treated as separate events in +// the buffer. +bool DtmfBuffer::SameEvent(const DtmfEvent& a, const DtmfEvent& b) { + return (a.event_no == b.event_no) && (a.timestamp == b.timestamp); +} + +bool DtmfBuffer::MergeEvents(DtmfList::iterator it, const DtmfEvent& event) { + if (SameEvent(*it, event)) { + if (!it->end_bit) { + // Do not extend the duration of an event for which the end bit was + // already received. + it->duration = std::max(event.duration, it->duration); + } + if (event.end_bit) { + it->end_bit = true; + } + return true; + } else { + return false; + } +} + +// Returns true if `a` goes before `b` in the sorting order ("`a` < `b`"). +// The events are ranked using their start timestamp (taking wrap-around into +// account). In the unlikely situation that two events share the same start +// timestamp, the event number is used to rank the two. Note that packets +// that belong to the same events, and therefore sharing the same start +// timestamp, have already been merged before the sort method is called. +bool DtmfBuffer::CompareEvents(const DtmfEvent& a, const DtmfEvent& b) { + if (a.timestamp == b.timestamp) { + return a.event_no < b.event_no; + } + // Take wrap-around into account. + return (static_cast(b.timestamp - a.timestamp) < 0xFFFFFFFF / 2); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.h b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.h new file mode 100644 index 0000000000..62b751525c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DTMF_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_DTMF_BUFFER_H_ + +#include +#include + +#include + +namespace webrtc { + +struct DtmfEvent { + uint32_t timestamp; + int event_no; + int volume; + int duration; + bool end_bit; + + // Constructors + DtmfEvent() + : timestamp(0), event_no(0), volume(0), duration(0), end_bit(false) {} + DtmfEvent(uint32_t ts, int ev, int vol, int dur, bool end) + : timestamp(ts), event_no(ev), volume(vol), duration(dur), end_bit(end) {} +}; + +// This is the buffer holding DTMF events while waiting for them to be played. +class DtmfBuffer { + public: + enum BufferReturnCodes { + kOK = 0, + kInvalidPointer, + kPayloadTooShort, + kInvalidEventParameters, + kInvalidSampleRate + }; + + // Set up the buffer for use at sample rate `fs_hz`. + explicit DtmfBuffer(int fs_hz); + + virtual ~DtmfBuffer(); + + DtmfBuffer(const DtmfBuffer&) = delete; + DtmfBuffer& operator=(const DtmfBuffer&) = delete; + + // Flushes the buffer. + virtual void Flush(); + + // Static method to parse 4 bytes from `payload` as a DTMF event (RFC 4733) + // and write the parsed information into the struct `event`. Input variable + // `rtp_timestamp` is simply copied into the struct. + static int ParseEvent(uint32_t rtp_timestamp, + const uint8_t* payload, + size_t payload_length_bytes, + DtmfEvent* event); + + // Inserts `event` into the buffer. The method looks for a matching event and + // merges the two if a match is found. + virtual int InsertEvent(const DtmfEvent& event); + + // Checks if a DTMF event should be played at time `current_timestamp`. If so, + // the method returns true; otherwise false. The parameters of the event to + // play will be written to `event`. + virtual bool GetEvent(uint32_t current_timestamp, DtmfEvent* event); + + // Number of events in the buffer. + virtual size_t Length() const; + + virtual bool Empty() const; + + // Set a new sample rate. + virtual int SetSampleRate(int fs_hz); + + private: + typedef std::list DtmfList; + + int max_extrapolation_samples_; + int frame_len_samples_; // TODO(hlundin): Remove this later. + + // Compares two events and returns true if they are the same. + static bool SameEvent(const DtmfEvent& a, const DtmfEvent& b); + + // Merges `event` to the event pointed out by `it`. The method checks that + // the two events are the same (using the SameEvent method), and merges them + // if that was the case, returning true. If the events are not the same, false + // is returned. + bool MergeEvents(DtmfList::iterator it, const DtmfEvent& event); + + // Method used by the sort algorithm to rank events in the buffer. + static bool CompareEvents(const DtmfEvent& a, const DtmfEvent& b); + + DtmfList buffer_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DTMF_BUFFER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer_unittest.cc new file mode 100644 index 0000000000..83745b6c09 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer_unittest.cc @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/dtmf_buffer.h" + +#ifdef WIN32 +#include // ntohl() +#else +#include // ntohl() +#endif + +#include + +#include "test/gtest.h" + +// Modify the tests so that they pass with the modifications done to DtmfBuffer +// for backwards bit-exactness. Once bit-exactness is no longer required, this +// #define should be removed (and the code that it enables). +#define LEGACY_BITEXACT + +namespace webrtc { + +static int sample_rate_hz = 8000; + +static uint32_t MakeDtmfPayload(int event, bool end, int volume, int duration) { + uint32_t payload = 0; + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | event |E|R| volume | duration | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + payload |= (event & 0x00FF) << 24; + payload |= (end ? 0x00800000 : 0x00000000); + payload |= (volume & 0x003F) << 16; + payload |= (duration & 0xFFFF); + payload = ntohl(payload); + return payload; +} + +static bool EqualEvents(const DtmfEvent& a, const DtmfEvent& b) { + return (a.duration == b.duration && a.end_bit == b.end_bit && + a.event_no == b.event_no && a.timestamp == b.timestamp && + a.volume == b.volume); +} + +TEST(DtmfBuffer, CreateAndDestroy) { + DtmfBuffer* buffer = new DtmfBuffer(sample_rate_hz); + delete buffer; +} + +// Test the event parser. +TEST(DtmfBuffer, ParseEvent) { + int event_no = 7; + bool end_bit = true; + int volume = 17; + int duration = 4711; + uint32_t timestamp = 0x12345678; + uint32_t payload = MakeDtmfPayload(event_no, end_bit, volume, duration); + uint8_t* payload_ptr = reinterpret_cast(&payload); + DtmfEvent event; + EXPECT_EQ(DtmfBuffer::kOK, DtmfBuffer::ParseEvent(timestamp, payload_ptr, + sizeof(payload), &event)); + EXPECT_EQ(duration, event.duration); + EXPECT_EQ(end_bit, event.end_bit); + EXPECT_EQ(event_no, event.event_no); + EXPECT_EQ(timestamp, event.timestamp); + EXPECT_EQ(volume, event.volume); + + EXPECT_EQ(DtmfBuffer::kPayloadTooShort, + DtmfBuffer::ParseEvent(timestamp, payload_ptr, 3, &event)); +} + +TEST(DtmfBuffer, SimpleInsertAndGet) { + int event_no = 7; + bool end_bit = true; + int volume = 17; + int duration = 4711; + uint32_t timestamp = 0x12345678; + DtmfEvent event(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); + EXPECT_EQ(1u, buffer.Length()); + EXPECT_FALSE(buffer.Empty()); + DtmfEvent out_event; + // Too early to get event. + EXPECT_FALSE(buffer.GetEvent(timestamp - 10, &out_event)); + EXPECT_EQ(1u, buffer.Length()); + EXPECT_FALSE(buffer.Empty()); + // Get the event at its starting timestamp. + EXPECT_TRUE(buffer.GetEvent(timestamp, &out_event)); + EXPECT_TRUE(EqualEvents(event, out_event)); + EXPECT_EQ(1u, buffer.Length()); + EXPECT_FALSE(buffer.Empty()); + // Get the event some time into the event. + EXPECT_TRUE(buffer.GetEvent(timestamp + duration / 2, &out_event)); + EXPECT_TRUE(EqualEvents(event, out_event)); + EXPECT_EQ(1u, buffer.Length()); + EXPECT_FALSE(buffer.Empty()); +// Give a "current" timestamp after the event has ended. +#ifdef LEGACY_BITEXACT + EXPECT_TRUE(buffer.GetEvent(timestamp + duration + 10, &out_event)); +#endif + EXPECT_FALSE(buffer.GetEvent(timestamp + duration + 10, &out_event)); + EXPECT_EQ(0u, buffer.Length()); + EXPECT_TRUE(buffer.Empty()); +} + +TEST(DtmfBuffer, MergingPackets) { + int event_no = 0; + bool end_bit = false; + int volume = 17; + int duration = 80; + uint32_t timestamp = 0x12345678; + DtmfEvent event(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); + + event.duration += 80; + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); + + event.duration += 80; + event.end_bit = true; + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); + + EXPECT_EQ(1u, buffer.Length()); + + DtmfEvent out_event; + EXPECT_TRUE(buffer.GetEvent(timestamp, &out_event)); + EXPECT_TRUE(EqualEvents(event, out_event)); +} + +// This test case inserts one shorter event completely overlapped by one longer +// event. The expected outcome is that only the longer event is played. +TEST(DtmfBuffer, OverlappingEvents) { + int event_no = 0; + bool end_bit = true; + int volume = 1; + int duration = 80; + uint32_t timestamp = 0x12345678 + 80; + DtmfEvent short_event(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(short_event)); + + event_no = 10; + end_bit = false; + timestamp = 0x12345678; + DtmfEvent long_event(timestamp, event_no, volume, duration, end_bit); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(long_event)); + + long_event.duration += 80; + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(long_event)); + + long_event.duration += 80; + long_event.end_bit = true; + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(long_event)); + + EXPECT_EQ(2u, buffer.Length()); + + DtmfEvent out_event; + // Expect to get the long event. + EXPECT_TRUE(buffer.GetEvent(timestamp, &out_event)); + EXPECT_TRUE(EqualEvents(long_event, out_event)); +// Expect no more events. +#ifdef LEGACY_BITEXACT + EXPECT_TRUE( + buffer.GetEvent(timestamp + long_event.duration + 10, &out_event)); + EXPECT_TRUE(EqualEvents(long_event, out_event)); + EXPECT_TRUE( + buffer.GetEvent(timestamp + long_event.duration + 10, &out_event)); + EXPECT_TRUE(EqualEvents(short_event, out_event)); +#else + EXPECT_FALSE( + buffer.GetEvent(timestamp + long_event.duration + 10, &out_event)); +#endif + EXPECT_TRUE(buffer.Empty()); +} + +TEST(DtmfBuffer, ExtrapolationTime) { + int event_no = 0; + bool end_bit = false; + int volume = 1; + int duration = 80; + uint32_t timestamp = 0x12345678; + DtmfEvent event1(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event1)); + EXPECT_EQ(1u, buffer.Length()); + + DtmfEvent out_event; + // Get the event at the start. + EXPECT_TRUE(buffer.GetEvent(timestamp, &out_event)); + EXPECT_TRUE(EqualEvents(event1, out_event)); + // Also get the event 100 samples after the end of the event (since we're + // missing the end bit). + uint32_t timestamp_now = timestamp + duration + 100; + EXPECT_TRUE(buffer.GetEvent(timestamp_now, &out_event)); + EXPECT_TRUE(EqualEvents(event1, out_event)); + // Insert another event starting back-to-back with the previous event. + timestamp += duration; + event_no = 1; + DtmfEvent event2(timestamp, event_no, volume, duration, end_bit); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event2)); + EXPECT_EQ(2u, buffer.Length()); + // Now we expect to get the new event when supplying `timestamp_now`. + EXPECT_TRUE(buffer.GetEvent(timestamp_now, &out_event)); + EXPECT_TRUE(EqualEvents(event2, out_event)); + // Expect the the first event to be erased now. + EXPECT_EQ(1u, buffer.Length()); + // Move `timestamp_now` to more than 560 samples after the end of the second + // event. Expect that event to be erased. + timestamp_now = timestamp + duration + 600; +#ifdef LEGACY_BITEXACT + EXPECT_TRUE(buffer.GetEvent(timestamp_now, &out_event)); +#endif + EXPECT_FALSE(buffer.GetEvent(timestamp_now, &out_event)); + EXPECT_TRUE(buffer.Empty()); +} + +TEST(DtmfBuffer, TimestampWraparound) { + int event_no = 0; + bool end_bit = true; + int volume = 1; + int duration = 80; + uint32_t timestamp1 = 0xFFFFFFFF - duration; + DtmfEvent event1(timestamp1, event_no, volume, duration, end_bit); + uint32_t timestamp2 = 0; + DtmfEvent event2(timestamp2, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event1)); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event2)); + EXPECT_EQ(2u, buffer.Length()); + DtmfEvent out_event; + EXPECT_TRUE(buffer.GetEvent(timestamp1, &out_event)); + EXPECT_TRUE(EqualEvents(event1, out_event)); +#ifdef LEGACY_BITEXACT + EXPECT_EQ(1u, buffer.Length()); +#else + EXPECT_EQ(2u, buffer.Length()); +#endif + + buffer.Flush(); + // Reverse the insert order. Expect same results. + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event2)); + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event1)); + EXPECT_EQ(2u, buffer.Length()); + EXPECT_TRUE(buffer.GetEvent(timestamp1, &out_event)); + EXPECT_TRUE(EqualEvents(event1, out_event)); +#ifdef LEGACY_BITEXACT + EXPECT_EQ(1u, buffer.Length()); +#else + EXPECT_EQ(2u, buffer.Length()); +#endif +} + +TEST(DtmfBuffer, InvalidEvents) { + int event_no = 0; + bool end_bit = true; + int volume = 1; + int duration = 80; + uint32_t timestamp = 0x12345678; + DtmfEvent event(timestamp, event_no, volume, duration, end_bit); + DtmfBuffer buffer(sample_rate_hz); + + // Invalid event number. + event.event_no = -1; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.event_no = 16; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.event_no = 0; // Valid value; + + // Invalid volume. + event.volume = -1; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.volume = 64; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.volume = 0; // Valid value; + + // Invalid duration. + event.duration = -1; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.duration = 0; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.duration = 0xFFFF + 1; + EXPECT_EQ(DtmfBuffer::kInvalidEventParameters, buffer.InsertEvent(event)); + event.duration = 1; // Valid value; + + // Finish with a valid event, just to verify that all is ok. + EXPECT_EQ(DtmfBuffer::kOK, buffer.InsertEvent(event)); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc new file mode 100644 index 0000000000..9061e27c67 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This class provides a generator for DTMF tones. The tone generation is based +// on a sinusoid recursion. Each sinusoid is generated using a recursion +// formula; x[n] = a * x[n-1] - x[n-2], where the coefficient +// a = 2*cos(2*pi*f/fs). The recursion is started with x[-1] = 0 and +// x[-2] = sin(2*pi*f/fs). (Note that with this initialization, the resulting +// sinusoid gets a "negative" rotation; x[n] = sin(-2*pi*f/fs * n + phi), but +// kept this way due to historical reasons.) +// TODO(hlundin): Change to positive rotation? +// +// Each key on the telephone keypad corresponds to an "event", 0-15. Each event +// is mapped to a tone pair, with a low and a high frequency. There are four +// low and four high frequencies, each corresponding to a row and column, +// respectively, on the keypad as illustrated below. +// +// 1209 Hz 1336 Hz 1477 Hz 1633 Hz +// 697 Hz 1 2 3 12 +// 770 Hz 4 5 6 13 +// 852 Hz 7 8 9 14 +// 941 Hz 10 0 11 15 + +#include "modules/audio_coding/neteq/dtmf_tone_generator.h" + +#include "modules/audio_coding/neteq/audio_vector.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// The filter coefficient a = 2*cos(2*pi*f/fs) for the low frequency tone, for +// sample rates fs = {8000, 16000, 32000, 48000} Hz, and events 0 through 15. +// Values are in Q14. +const int DtmfToneGenerator::kCoeff1[4][16] = { + {24219, 27980, 27980, 27980, 26956, 26956, 26956, 25701, 25701, 25701, + 24219, 24219, 27980, 26956, 25701, 24219}, + {30556, 31548, 31548, 31548, 31281, 31281, 31281, 30951, 30951, 30951, + 30556, 30556, 31548, 31281, 30951, 30556}, + {32210, 32462, 32462, 32462, 32394, 32394, 32394, 32311, 32311, 32311, + 32210, 32210, 32462, 32394, 32311, 32210}, + {32520, 32632, 32632, 32632, 32602, 32602, 32602, 32564, 32564, 32564, + 32520, 32520, 32632, 32602, 32564, 32520}}; + +// The filter coefficient a = 2*cos(2*pi*f/fs) for the high frequency tone, for +// sample rates fs = {8000, 16000, 32000, 48000} Hz, and events 0 through 15. +// Values are in Q14. +const int DtmfToneGenerator::kCoeff2[4][16] = { + {16325, 19073, 16325, 13085, 19073, 16325, 13085, 19073, 16325, 13085, + 19073, 13085, 9315, 9315, 9315, 9315}, + {28361, 29144, 28361, 27409, 29144, 28361, 27409, 29144, 28361, 27409, + 29144, 27409, 26258, 26258, 26258, 26258}, + {31647, 31849, 31647, 31400, 31849, 31647, 31400, 31849, 31647, 31400, + 31849, 31400, 31098, 31098, 31098, 31098}, + {32268, 32359, 32268, 32157, 32359, 32268, 32157, 32359, 32268, 32157, + 32359, 32157, 32022, 32022, 32022, 32022}}; + +// The initialization value x[-2] = sin(2*pi*f/fs) for the low frequency tone, +// for sample rates fs = {8000, 16000, 32000, 48000} Hz, and events 0-15. +// Values are in Q14. +const int DtmfToneGenerator::kInitValue1[4][16] = { + {11036, 8528, 8528, 8528, 9315, 9315, 9315, 10163, 10163, 10163, 11036, + 11036, 8528, 9315, 10163, 11036}, + {5918, 4429, 4429, 4429, 4879, 4879, 4879, 5380, 5380, 5380, 5918, 5918, + 4429, 4879, 5380, 5918}, + {3010, 2235, 2235, 2235, 2468, 2468, 2468, 2728, 2728, 2728, 3010, 3010, + 2235, 2468, 2728, 3010}, + {2013, 1493, 1493, 1493, 1649, 1649, 1649, 1823, 1823, 1823, 2013, 2013, + 1493, 1649, 1823, 2013}}; + +// The initialization value x[-2] = sin(2*pi*f/fs) for the high frequency tone, +// for sample rates fs = {8000, 16000, 32000, 48000} Hz, and events 0-15. +// Values are in Q14. +const int DtmfToneGenerator::kInitValue2[4][16] = { + {14206, 13323, 14206, 15021, 13323, 14206, 15021, 13323, 14206, 15021, + 13323, 15021, 15708, 15708, 15708, 15708}, + {8207, 7490, 8207, 8979, 7490, 8207, 8979, 7490, 8207, 8979, 7490, 8979, + 9801, 9801, 9801, 9801}, + {4249, 3853, 4249, 4685, 3853, 4249, 4685, 3853, 4249, 4685, 3853, 4685, + 5164, 5164, 5164, 5164}, + {2851, 2582, 2851, 3148, 2582, 2851, 3148, 2582, 2851, 3148, 2582, 3148, + 3476, 3476, 3476, 3476}}; + +// Amplitude multipliers for volume values 0 through 63, corresponding to +// 0 dBm0 through -63 dBm0. Values are in Q14. +// for a in range(0, 64): +// print round(16141.0 * 10**(-float(a)/20)) +const int DtmfToneGenerator::kAmplitude[64] = { + 16141, 14386, 12821, 11427, 10184, 9077, 8090, 7210, 6426, 5727, 5104, + 4549, 4054, 3614, 3221, 2870, 2558, 2280, 2032, 1811, 1614, 1439, + 1282, 1143, 1018, 908, 809, 721, 643, 573, 510, 455, 405, + 361, 322, 287, 256, 228, 203, 181, 161, 144, 128, 114, + 102, 91, 81, 72, 64, 57, 51, 45, 41, 36, 32, + 29, 26, 23, 20, 18, 16, 14, 13, 11}; + +// Constructor. +DtmfToneGenerator::DtmfToneGenerator() + : initialized_(false), coeff1_(0), coeff2_(0), amplitude_(0) {} + +// Initialize the DTMF generator with sample rate fs Hz (8000, 16000, 32000, +// 48000), event (0-15) and attenuation (0-36 dB). +// Returns 0 on success, otherwise an error code. +int DtmfToneGenerator::Init(int fs, int event, int attenuation) { + initialized_ = false; + size_t fs_index; + if (fs == 8000) { + fs_index = 0; + } else if (fs == 16000) { + fs_index = 1; + } else if (fs == 32000) { + fs_index = 2; + } else if (fs == 48000) { + fs_index = 3; + } else { + RTC_DCHECK_NOTREACHED(); + fs_index = 1; // Default to 8000 Hz. + } + + if (event < 0 || event > 15) { + return kParameterError; // Invalid event number. + } + + if (attenuation < 0 || attenuation > 63) { + return kParameterError; // Invalid attenuation. + } + + // Look up oscillator coefficient for low and high frequencies. + RTC_DCHECK_LE(0, fs_index); + RTC_DCHECK_GT(arraysize(kCoeff1), fs_index); + RTC_DCHECK_GT(arraysize(kCoeff2), fs_index); + RTC_DCHECK_LE(0, event); + RTC_DCHECK_GT(arraysize(kCoeff1[fs_index]), event); + RTC_DCHECK_GT(arraysize(kCoeff2[fs_index]), event); + coeff1_ = kCoeff1[fs_index][event]; + coeff2_ = kCoeff2[fs_index][event]; + + // Look up amplitude multiplier. + RTC_DCHECK_LE(0, attenuation); + RTC_DCHECK_GT(arraysize(kAmplitude), attenuation); + amplitude_ = kAmplitude[attenuation]; + + // Initialize sample history. + RTC_DCHECK_LE(0, fs_index); + RTC_DCHECK_GT(arraysize(kInitValue1), fs_index); + RTC_DCHECK_GT(arraysize(kInitValue2), fs_index); + RTC_DCHECK_LE(0, event); + RTC_DCHECK_GT(arraysize(kInitValue1[fs_index]), event); + RTC_DCHECK_GT(arraysize(kInitValue2[fs_index]), event); + sample_history1_[0] = kInitValue1[fs_index][event]; + sample_history1_[1] = 0; + sample_history2_[0] = kInitValue2[fs_index][event]; + sample_history2_[1] = 0; + + initialized_ = true; + return 0; +} + +// Reset tone generator to uninitialized state. +void DtmfToneGenerator::Reset() { + initialized_ = false; +} + +// Generate num_samples of DTMF signal and write to `output`. +int DtmfToneGenerator::Generate(size_t num_samples, AudioMultiVector* output) { + if (!initialized_) { + return kNotInitialized; + } + + if (!output) { + return kParameterError; + } + + output->AssertSize(num_samples); + for (size_t i = 0; i < num_samples; ++i) { + // Use recursion formula y[n] = a * y[n - 1] - y[n - 2]. + int16_t temp_val_low = + ((coeff1_ * sample_history1_[1] + 8192) >> 14) - sample_history1_[0]; + int16_t temp_val_high = + ((coeff2_ * sample_history2_[1] + 8192) >> 14) - sample_history2_[0]; + + // Update recursion memory. + sample_history1_[0] = sample_history1_[1]; + sample_history1_[1] = temp_val_low; + sample_history2_[0] = sample_history2_[1]; + sample_history2_[1] = temp_val_high; + + // Attenuate the low frequency tone 3 dB. + int32_t temp_val = + kAmpMultiplier * temp_val_low + temp_val_high * (1 << 15); + // Normalize the signal to Q14 with proper rounding. + temp_val = (temp_val + 16384) >> 15; + // Scale the signal to correct volume. + (*output)[0][i] = + static_cast((temp_val * amplitude_ + 8192) >> 14); + } + // Copy first channel to all other channels. + for (size_t channel = 1; channel < output->Channels(); ++channel) { + output->CopyChannel(0, channel); + } + + return static_cast(num_samples); +} + +bool DtmfToneGenerator::initialized() const { + return initialized_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.h b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.h new file mode 100644 index 0000000000..35114f4f49 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_DTMF_TONE_GENERATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_DTMF_TONE_GENERATOR_H_ + +#include +#include + +#include "modules/audio_coding/neteq/audio_multi_vector.h" + +namespace webrtc { + +// This class provides a generator for DTMF tones. +class DtmfToneGenerator { + public: + enum ReturnCodes { + kNotInitialized = -1, + kParameterError = -2, + }; + + DtmfToneGenerator(); + virtual ~DtmfToneGenerator() {} + + DtmfToneGenerator(const DtmfToneGenerator&) = delete; + DtmfToneGenerator& operator=(const DtmfToneGenerator&) = delete; + + virtual int Init(int fs, int event, int attenuation); + virtual void Reset(); + virtual int Generate(size_t num_samples, AudioMultiVector* output); + virtual bool initialized() const; + + private: + static const int kCoeff1[4][16]; // 1st oscillator model coefficient table. + static const int kCoeff2[4][16]; // 2nd oscillator model coefficient table. + static const int kInitValue1[4][16]; // Initialization for 1st oscillator. + static const int kInitValue2[4][16]; // Initialization for 2nd oscillator. + static const int kAmplitude[64]; // Amplitude for 0 through -63 dBm0. + static const int16_t kAmpMultiplier = 23171; // 3 dB attenuation (in Q15). + + bool initialized_; // True if generator is initialized properly. + int coeff1_; // 1st oscillator coefficient for this event. + int coeff2_; // 2nd oscillator coefficient for this event. + int amplitude_; // Amplitude for this event. + int16_t sample_history1_[2]; // Last 2 samples for the 1st oscillator. + int16_t sample_history2_[2]; // Last 2 samples for the 2nd oscillator. +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_DTMF_TONE_GENERATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator_unittest.cc new file mode 100644 index 0000000000..e843706dd3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator_unittest.cc @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for DtmfToneGenerator class. + +#include "modules/audio_coding/neteq/dtmf_tone_generator.h" + +#include + +#include "common_audio/include/audio_util.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" + +namespace webrtc { + +class DtmfToneGeneratorTest : public ::testing::Test { + protected: + static const double kLowFreqHz[16]; + static const double kHighFreqHz[16]; + // This is the attenuation applied to all cases. + const double kBaseAttenuation = 16141.0 / 16384.0; + const double k3dbAttenuation = 23171.0 / 32768; + const int kNumSamples = 10; + + void TestAllTones(int fs_hz, int channels) { + AudioMultiVector signal(channels); + + for (int event = 0; event <= 15; ++event) { + rtc::StringBuilder ss; + ss << "Checking event " << event << " at sample rate " << fs_hz; + SCOPED_TRACE(ss.str()); + const int kAttenuation = 0; + ASSERT_EQ(0, tone_gen_.Init(fs_hz, event, kAttenuation)); + EXPECT_TRUE(tone_gen_.initialized()); + EXPECT_EQ(kNumSamples, tone_gen_.Generate(kNumSamples, &signal)); + + double f1 = kLowFreqHz[event]; + double f2 = kHighFreqHz[event]; + const double pi = 3.14159265358979323846; + + for (int n = 0; n < kNumSamples; ++n) { + double x = k3dbAttenuation * sin(2.0 * pi * f1 / fs_hz * (-n - 1)) + + sin(2.0 * pi * f2 / fs_hz * (-n - 1)); + x *= kBaseAttenuation; + x = ldexp(x, 14); // Scale to Q14. + for (int channel = 0; channel < channels; ++channel) { + EXPECT_NEAR(x, static_cast(signal[channel][n]), 25); + } + } + + tone_gen_.Reset(); + EXPECT_FALSE(tone_gen_.initialized()); + } + } + + void TestAmplitudes(int fs_hz, int channels) { + AudioMultiVector signal(channels); + AudioMultiVector ref_signal(channels); + + const int event_vec[] = {0, 4, 9, 13}; // Test a few events. + for (int e = 0; e < 4; ++e) { + int event = event_vec[e]; + // Create full-scale reference. + ASSERT_EQ(0, tone_gen_.Init(fs_hz, event, 0)); // 0 attenuation. + EXPECT_EQ(kNumSamples, tone_gen_.Generate(kNumSamples, &ref_signal)); + // Test every 5 steps (to save time). + for (int attenuation = 1; attenuation <= 63; attenuation += 5) { + rtc::StringBuilder ss; + ss << "Checking event " << event << " at sample rate " << fs_hz; + ss << "; attenuation " << attenuation; + SCOPED_TRACE(ss.str()); + ASSERT_EQ(0, tone_gen_.Init(fs_hz, event, attenuation)); + EXPECT_EQ(kNumSamples, tone_gen_.Generate(kNumSamples, &signal)); + for (int n = 0; n < kNumSamples; ++n) { + double attenuation_factor = + DbToRatio(-static_cast(attenuation)); + // Verify that the attenuation is correct. + for (int channel = 0; channel < channels; ++channel) { + EXPECT_NEAR(attenuation_factor * ref_signal[channel][n], + signal[channel][n], 2); + } + } + + tone_gen_.Reset(); + } + } + } + + DtmfToneGenerator tone_gen_; +}; + +// Low and high frequencies for events 0 through 15. +const double DtmfToneGeneratorTest::kLowFreqHz[16] = { + 941.0, 697.0, 697.0, 697.0, 770.0, 770.0, 770.0, 852.0, + 852.0, 852.0, 941.0, 941.0, 697.0, 770.0, 852.0, 941.0}; +const double DtmfToneGeneratorTest::kHighFreqHz[16] = { + 1336.0, 1209.0, 1336.0, 1477.0, 1209.0, 1336.0, 1477.0, 1209.0, + 1336.0, 1477.0, 1209.0, 1477.0, 1633.0, 1633.0, 1633.0, 1633.0}; + +TEST_F(DtmfToneGeneratorTest, Test8000Mono) { + TestAllTones(8000, 1); + TestAmplitudes(8000, 1); +} + +TEST_F(DtmfToneGeneratorTest, Test16000Mono) { + TestAllTones(16000, 1); + TestAmplitudes(16000, 1); +} + +TEST_F(DtmfToneGeneratorTest, Test32000Mono) { + TestAllTones(32000, 1); + TestAmplitudes(32000, 1); +} + +TEST_F(DtmfToneGeneratorTest, Test48000Mono) { + TestAllTones(48000, 1); + TestAmplitudes(48000, 1); +} + +TEST_F(DtmfToneGeneratorTest, Test8000Stereo) { + TestAllTones(8000, 2); + TestAmplitudes(8000, 2); +} + +TEST_F(DtmfToneGeneratorTest, Test16000Stereo) { + TestAllTones(16000, 2); + TestAmplitudes(16000, 2); +} + +TEST_F(DtmfToneGeneratorTest, Test32000Stereo) { + TestAllTones(32000, 2); + TestAmplitudes(32000, 2); +} + +TEST_F(DtmfToneGeneratorTest, Test48000Stereo) { + TestAllTones(48000, 2); + TestAmplitudes(48000, 2); +} + +TEST(DtmfToneGenerator, TestErrors) { + DtmfToneGenerator tone_gen; + const int kNumSamples = 10; + AudioMultiVector signal(1); // One channel. + + // Try to generate tones without initializing. + EXPECT_EQ(DtmfToneGenerator::kNotInitialized, + tone_gen.Generate(kNumSamples, &signal)); + + const int fs = 16000; // Valid sample rate. + const int event = 7; // Valid event. + const int attenuation = 0; // Valid attenuation. + // Initialize with invalid event -1. + EXPECT_EQ(DtmfToneGenerator::kParameterError, + tone_gen.Init(fs, -1, attenuation)); + // Initialize with invalid event 16. + EXPECT_EQ(DtmfToneGenerator::kParameterError, + tone_gen.Init(fs, 16, attenuation)); + // Initialize with invalid attenuation -1. + EXPECT_EQ(DtmfToneGenerator::kParameterError, tone_gen.Init(fs, event, -1)); + // Initialize with invalid attenuation 64. + EXPECT_EQ(DtmfToneGenerator::kParameterError, tone_gen.Init(fs, event, 64)); + EXPECT_FALSE(tone_gen.initialized()); // Should still be uninitialized. + + // Initialize with valid parameters. + ASSERT_EQ(0, tone_gen.Init(fs, event, attenuation)); + EXPECT_TRUE(tone_gen.initialized()); + // NULL pointer to destination. + EXPECT_EQ(DtmfToneGenerator::kParameterError, + tone_gen.Generate(kNumSamples, NULL)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/expand.cc b/third_party/libwebrtc/modules/audio_coding/neteq/expand.cc new file mode 100644 index 0000000000..9c3274609f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/expand.cc @@ -0,0 +1,888 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/expand.h" + +#include // memset + +#include // min, max +#include // numeric_limits + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/cross_correlation.h" +#include "modules/audio_coding/neteq/dsp_helper.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +Expand::Expand(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels) + : random_vector_(random_vector), + sync_buffer_(sync_buffer), + first_expand_(true), + fs_hz_(fs), + num_channels_(num_channels), + consecutive_expands_(0), + background_noise_(background_noise), + statistics_(statistics), + overlap_length_(5 * fs / 8000), + lag_index_direction_(0), + current_lag_index_(0), + stop_muting_(false), + expand_duration_samples_(0), + channel_parameters_(new ChannelParameters[num_channels_]) { + RTC_DCHECK(fs == 8000 || fs == 16000 || fs == 32000 || fs == 48000); + RTC_DCHECK_LE(fs, + static_cast(kMaxSampleRate)); // Should not be possible. + RTC_DCHECK_GT(num_channels_, 0); + memset(expand_lags_, 0, sizeof(expand_lags_)); + Reset(); +} + +Expand::~Expand() = default; + +void Expand::Reset() { + first_expand_ = true; + consecutive_expands_ = 0; + max_lag_ = 0; + for (size_t ix = 0; ix < num_channels_; ++ix) { + channel_parameters_[ix].expand_vector0.Clear(); + channel_parameters_[ix].expand_vector1.Clear(); + } +} + +int Expand::Process(AudioMultiVector* output) { + int16_t random_vector[kMaxSampleRate / 8000 * 120 + 30]; + int16_t scaled_random_vector[kMaxSampleRate / 8000 * 125]; + static const int kTempDataSize = 3600; + int16_t temp_data[kTempDataSize]; // TODO(hlundin) Remove this. + int16_t* voiced_vector_storage = temp_data; + int16_t* voiced_vector = &voiced_vector_storage[overlap_length_]; + static const size_t kNoiseLpcOrder = BackgroundNoise::kMaxLpcOrder; + int16_t unvoiced_array_memory[kNoiseLpcOrder + kMaxSampleRate / 8000 * 125]; + int16_t* unvoiced_vector = unvoiced_array_memory + kUnvoicedLpcOrder; + int16_t* noise_vector = unvoiced_array_memory + kNoiseLpcOrder; + + int fs_mult = fs_hz_ / 8000; + + if (first_expand_) { + // Perform initial setup if this is the first expansion since last reset. + AnalyzeSignal(random_vector); + first_expand_ = false; + expand_duration_samples_ = 0; + } else { + // This is not the first expansion, parameters are already estimated. + // Extract a noise segment. + size_t rand_length = max_lag_; + // This only applies to SWB where length could be larger than 256. + RTC_DCHECK_LE(rand_length, kMaxSampleRate / 8000 * 120 + 30); + GenerateRandomVector(2, rand_length, random_vector); + } + + // Generate signal. + UpdateLagIndex(); + + // Voiced part. + // Generate a weighted vector with the current lag. + size_t expansion_vector_length = max_lag_ + overlap_length_; + size_t current_lag = expand_lags_[current_lag_index_]; + // Copy lag+overlap data. + size_t expansion_vector_position = + expansion_vector_length - current_lag - overlap_length_; + size_t temp_length = current_lag + overlap_length_; + for (size_t channel_ix = 0; channel_ix < num_channels_; ++channel_ix) { + ChannelParameters& parameters = channel_parameters_[channel_ix]; + if (current_lag_index_ == 0) { + // Use only expand_vector0. + RTC_DCHECK_LE(expansion_vector_position + temp_length, + parameters.expand_vector0.Size()); + parameters.expand_vector0.CopyTo(temp_length, expansion_vector_position, + voiced_vector_storage); + } else if (current_lag_index_ == 1) { + std::unique_ptr temp_0(new int16_t[temp_length]); + parameters.expand_vector0.CopyTo(temp_length, expansion_vector_position, + temp_0.get()); + std::unique_ptr temp_1(new int16_t[temp_length]); + parameters.expand_vector1.CopyTo(temp_length, expansion_vector_position, + temp_1.get()); + // Mix 3/4 of expand_vector0 with 1/4 of expand_vector1. + WebRtcSpl_ScaleAndAddVectorsWithRound(temp_0.get(), 3, temp_1.get(), 1, 2, + voiced_vector_storage, temp_length); + } else if (current_lag_index_ == 2) { + // Mix 1/2 of expand_vector0 with 1/2 of expand_vector1. + RTC_DCHECK_LE(expansion_vector_position + temp_length, + parameters.expand_vector0.Size()); + RTC_DCHECK_LE(expansion_vector_position + temp_length, + parameters.expand_vector1.Size()); + + std::unique_ptr temp_0(new int16_t[temp_length]); + parameters.expand_vector0.CopyTo(temp_length, expansion_vector_position, + temp_0.get()); + std::unique_ptr temp_1(new int16_t[temp_length]); + parameters.expand_vector1.CopyTo(temp_length, expansion_vector_position, + temp_1.get()); + WebRtcSpl_ScaleAndAddVectorsWithRound(temp_0.get(), 1, temp_1.get(), 1, 1, + voiced_vector_storage, temp_length); + } + + // Get tapering window parameters. Values are in Q15. + int16_t muting_window, muting_window_increment; + int16_t unmuting_window, unmuting_window_increment; + if (fs_hz_ == 8000) { + muting_window = DspHelper::kMuteFactorStart8kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement8kHz; + unmuting_window = DspHelper::kUnmuteFactorStart8kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement8kHz; + } else if (fs_hz_ == 16000) { + muting_window = DspHelper::kMuteFactorStart16kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement16kHz; + unmuting_window = DspHelper::kUnmuteFactorStart16kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement16kHz; + } else if (fs_hz_ == 32000) { + muting_window = DspHelper::kMuteFactorStart32kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement32kHz; + unmuting_window = DspHelper::kUnmuteFactorStart32kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement32kHz; + } else { // fs_ == 48000 + muting_window = DspHelper::kMuteFactorStart48kHz; + muting_window_increment = DspHelper::kMuteFactorIncrement48kHz; + unmuting_window = DspHelper::kUnmuteFactorStart48kHz; + unmuting_window_increment = DspHelper::kUnmuteFactorIncrement48kHz; + } + + // Smooth the expanded if it has not been muted to a low amplitude and + // `current_voice_mix_factor` is larger than 0.5. + if ((parameters.mute_factor > 819) && + (parameters.current_voice_mix_factor > 8192)) { + size_t start_ix = sync_buffer_->Size() - overlap_length_; + for (size_t i = 0; i < overlap_length_; i++) { + // Do overlap add between new vector and overlap. + (*sync_buffer_)[channel_ix][start_ix + i] = + (((*sync_buffer_)[channel_ix][start_ix + i] * muting_window) + + (((parameters.mute_factor * voiced_vector_storage[i]) >> 14) * + unmuting_window) + + 16384) >> + 15; + muting_window += muting_window_increment; + unmuting_window += unmuting_window_increment; + } + } else if (parameters.mute_factor == 0) { + // The expanded signal will consist of only comfort noise if + // mute_factor = 0. Set the output length to 15 ms for best noise + // production. + // TODO(hlundin): This has been disabled since the length of + // parameters.expand_vector0 and parameters.expand_vector1 no longer + // match with expand_lags_, causing invalid reads and writes. Is it a good + // idea to enable this again, and solve the vector size problem? + // max_lag_ = fs_mult * 120; + // expand_lags_[0] = fs_mult * 120; + // expand_lags_[1] = fs_mult * 120; + // expand_lags_[2] = fs_mult * 120; + } + + // Unvoiced part. + // Filter `scaled_random_vector` through `ar_filter_`. + memcpy(unvoiced_vector - kUnvoicedLpcOrder, parameters.ar_filter_state, + sizeof(int16_t) * kUnvoicedLpcOrder); + int32_t add_constant = 0; + if (parameters.ar_gain_scale > 0) { + add_constant = 1 << (parameters.ar_gain_scale - 1); + } + WebRtcSpl_AffineTransformVector(scaled_random_vector, random_vector, + parameters.ar_gain, add_constant, + parameters.ar_gain_scale, current_lag); + WebRtcSpl_FilterARFastQ12(scaled_random_vector, unvoiced_vector, + parameters.ar_filter, kUnvoicedLpcOrder + 1, + current_lag); + memcpy(parameters.ar_filter_state, + &(unvoiced_vector[current_lag - kUnvoicedLpcOrder]), + sizeof(int16_t) * kUnvoicedLpcOrder); + + // Combine voiced and unvoiced contributions. + + // Set a suitable cross-fading slope. + // For lag = + // <= 31 * fs_mult => go from 1 to 0 in about 8 ms; + // (>= 31 .. <= 63) * fs_mult => go from 1 to 0 in about 16 ms; + // >= 64 * fs_mult => go from 1 to 0 in about 32 ms. + // temp_shift = getbits(max_lag_) - 5. + int temp_shift = + (31 - WebRtcSpl_NormW32(rtc::dchecked_cast(max_lag_))) - 5; + int16_t mix_factor_increment = 256 >> temp_shift; + if (stop_muting_) { + mix_factor_increment = 0; + } + + // Create combined signal by shifting in more and more of unvoiced part. + temp_shift = 8 - temp_shift; // = getbits(mix_factor_increment). + size_t temp_length = + (parameters.current_voice_mix_factor - parameters.voice_mix_factor) >> + temp_shift; + temp_length = std::min(temp_length, current_lag); + DspHelper::CrossFade(voiced_vector, unvoiced_vector, temp_length, + ¶meters.current_voice_mix_factor, + mix_factor_increment, temp_data); + + // End of cross-fading period was reached before end of expanded signal + // path. Mix the rest with a fixed mixing factor. + if (temp_length < current_lag) { + if (mix_factor_increment != 0) { + parameters.current_voice_mix_factor = parameters.voice_mix_factor; + } + int16_t temp_scale = 16384 - parameters.current_voice_mix_factor; + WebRtcSpl_ScaleAndAddVectorsWithRound( + voiced_vector + temp_length, parameters.current_voice_mix_factor, + unvoiced_vector + temp_length, temp_scale, 14, + temp_data + temp_length, current_lag - temp_length); + } + + // Select muting slope depending on how many consecutive expands we have + // done. + if (consecutive_expands_ == 3) { + // Let the mute factor decrease from 1.0 to 0.95 in 6.25 ms. + // mute_slope = 0.0010 / fs_mult in Q20. + parameters.mute_slope = std::max(parameters.mute_slope, 1049 / fs_mult); + } + if (consecutive_expands_ == 7) { + // Let the mute factor decrease from 1.0 to 0.90 in 6.25 ms. + // mute_slope = 0.0020 / fs_mult in Q20. + parameters.mute_slope = std::max(parameters.mute_slope, 2097 / fs_mult); + } + + // Mute segment according to slope value. + if ((consecutive_expands_ != 0) || !parameters.onset) { + // Mute to the previous level, then continue with the muting. + WebRtcSpl_AffineTransformVector( + temp_data, temp_data, parameters.mute_factor, 8192, 14, current_lag); + + if (!stop_muting_) { + DspHelper::MuteSignal(temp_data, parameters.mute_slope, current_lag); + + // Shift by 6 to go from Q20 to Q14. + // TODO(hlundin): Adding 8192 before shifting 6 steps seems wrong. + // Legacy. + int16_t gain = static_cast( + 16384 - (((current_lag * parameters.mute_slope) + 8192) >> 6)); + gain = ((gain * parameters.mute_factor) + 8192) >> 14; + + // Guard against getting stuck with very small (but sometimes audible) + // gain. + if ((consecutive_expands_ > 3) && (gain >= parameters.mute_factor)) { + parameters.mute_factor = 0; + } else { + parameters.mute_factor = gain; + } + } + } + + // Background noise part. + background_noise_->GenerateBackgroundNoise( + random_vector, channel_ix, channel_parameters_[channel_ix].mute_slope, + TooManyExpands(), current_lag, unvoiced_array_memory); + + // Add background noise to the combined voiced-unvoiced signal. + for (size_t i = 0; i < current_lag; i++) { + temp_data[i] = temp_data[i] + noise_vector[i]; + } + if (channel_ix == 0) { + output->AssertSize(current_lag); + } else { + RTC_DCHECK_EQ(output->Size(), current_lag); + } + (*output)[channel_ix].OverwriteAt(temp_data, current_lag, 0); + } + + // Increase call number and cap it. + consecutive_expands_ = consecutive_expands_ >= kMaxConsecutiveExpands + ? kMaxConsecutiveExpands + : consecutive_expands_ + 1; + expand_duration_samples_ += output->Size(); + // Clamp the duration counter at 2 seconds. + expand_duration_samples_ = std::min(expand_duration_samples_, + rtc::dchecked_cast(fs_hz_ * 2)); + return 0; +} + +void Expand::SetParametersForNormalAfterExpand() { + current_lag_index_ = 0; + lag_index_direction_ = 0; + stop_muting_ = true; // Do not mute signal any more. + statistics_->LogDelayedPacketOutageEvent(expand_duration_samples_, fs_hz_); + statistics_->EndExpandEvent(fs_hz_); +} + +void Expand::SetParametersForMergeAfterExpand() { + current_lag_index_ = -1; /* out of the 3 possible ones */ + lag_index_direction_ = 1; /* make sure we get the "optimal" lag */ + stop_muting_ = true; + statistics_->EndExpandEvent(fs_hz_); +} + +bool Expand::Muted() const { + if (first_expand_ || stop_muting_) + return false; + RTC_DCHECK(channel_parameters_); + for (size_t ch = 0; ch < num_channels_; ++ch) { + if (channel_parameters_[ch].mute_factor != 0) + return false; + } + return true; +} + +size_t Expand::overlap_length() const { + return overlap_length_; +} + +void Expand::InitializeForAnExpandPeriod() { + lag_index_direction_ = 1; + current_lag_index_ = -1; + stop_muting_ = false; + random_vector_->set_seed_increment(1); + consecutive_expands_ = 0; + for (size_t ix = 0; ix < num_channels_; ++ix) { + channel_parameters_[ix].current_voice_mix_factor = 16384; // 1.0 in Q14. + channel_parameters_[ix].mute_factor = 16384; // 1.0 in Q14. + // Start with 0 gain for background noise. + background_noise_->SetMuteFactor(ix, 0); + } +} + +bool Expand::TooManyExpands() { + return consecutive_expands_ >= kMaxConsecutiveExpands; +} + +void Expand::AnalyzeSignal(int16_t* random_vector) { + int32_t auto_correlation[kUnvoicedLpcOrder + 1]; + int16_t reflection_coeff[kUnvoicedLpcOrder]; + int16_t correlation_vector[kMaxSampleRate / 8000 * 102]; + size_t best_correlation_index[kNumCorrelationCandidates]; + int16_t best_correlation[kNumCorrelationCandidates]; + size_t best_distortion_index[kNumCorrelationCandidates]; + int16_t best_distortion[kNumCorrelationCandidates]; + int32_t correlation_vector2[(99 * kMaxSampleRate / 8000) + 1]; + int32_t best_distortion_w32[kNumCorrelationCandidates]; + static const size_t kNoiseLpcOrder = BackgroundNoise::kMaxLpcOrder; + int16_t unvoiced_array_memory[kNoiseLpcOrder + kMaxSampleRate / 8000 * 125]; + int16_t* unvoiced_vector = unvoiced_array_memory + kUnvoicedLpcOrder; + + int fs_mult = fs_hz_ / 8000; + + // Pre-calculate common multiplications with fs_mult. + size_t fs_mult_4 = static_cast(fs_mult * 4); + size_t fs_mult_20 = static_cast(fs_mult * 20); + size_t fs_mult_120 = static_cast(fs_mult * 120); + size_t fs_mult_dist_len = fs_mult * kDistortionLength; + size_t fs_mult_lpc_analysis_len = fs_mult * kLpcAnalysisLength; + + const size_t signal_length = static_cast(256 * fs_mult); + + const size_t audio_history_position = sync_buffer_->Size() - signal_length; + std::unique_ptr audio_history(new int16_t[signal_length]); + (*sync_buffer_)[0].CopyTo(signal_length, audio_history_position, + audio_history.get()); + + // Initialize. + InitializeForAnExpandPeriod(); + + // Calculate correlation in downsampled domain (4 kHz sample rate). + size_t correlation_length = 51; // TODO(hlundin): Legacy bit-exactness. + // If it is decided to break bit-exactness `correlation_length` should be + // initialized to the return value of Correlation(). + Correlation(audio_history.get(), signal_length, correlation_vector); + + // Find peaks in correlation vector. + DspHelper::PeakDetection(correlation_vector, correlation_length, + kNumCorrelationCandidates, fs_mult, + best_correlation_index, best_correlation); + + // Adjust peak locations; cross-correlation lags start at 2.5 ms + // (20 * fs_mult samples). + best_correlation_index[0] += fs_mult_20; + best_correlation_index[1] += fs_mult_20; + best_correlation_index[2] += fs_mult_20; + + // Calculate distortion around the `kNumCorrelationCandidates` best lags. + int distortion_scale = 0; + for (size_t i = 0; i < kNumCorrelationCandidates; i++) { + size_t min_index = + std::max(fs_mult_20, best_correlation_index[i] - fs_mult_4); + size_t max_index = + std::min(fs_mult_120 - 1, best_correlation_index[i] + fs_mult_4); + best_distortion_index[i] = DspHelper::MinDistortion( + &(audio_history[signal_length - fs_mult_dist_len]), min_index, + max_index, fs_mult_dist_len, &best_distortion_w32[i]); + distortion_scale = std::max(16 - WebRtcSpl_NormW32(best_distortion_w32[i]), + distortion_scale); + } + // Shift the distortion values to fit in 16 bits. + WebRtcSpl_VectorBitShiftW32ToW16(best_distortion, kNumCorrelationCandidates, + best_distortion_w32, distortion_scale); + + // Find the maximizing index `i` of the cost function + // f[i] = best_correlation[i] / best_distortion[i]. + int32_t best_ratio = std::numeric_limits::min(); + size_t best_index = std::numeric_limits::max(); + for (size_t i = 0; i < kNumCorrelationCandidates; ++i) { + int32_t ratio; + if (best_distortion[i] > 0) { + ratio = (best_correlation[i] * (1 << 16)) / best_distortion[i]; + } else if (best_correlation[i] == 0) { + ratio = 0; // No correlation set result to zero. + } else { + ratio = std::numeric_limits::max(); // Denominator is zero. + } + if (ratio > best_ratio) { + best_index = i; + best_ratio = ratio; + } + } + + size_t distortion_lag = best_distortion_index[best_index]; + size_t correlation_lag = best_correlation_index[best_index]; + max_lag_ = std::max(distortion_lag, correlation_lag); + + // Calculate the exact best correlation in the range between + // `correlation_lag` and `distortion_lag`. + correlation_length = std::max(std::min(distortion_lag + 10, fs_mult_120), + static_cast(60 * fs_mult)); + + size_t start_index = std::min(distortion_lag, correlation_lag); + size_t correlation_lags = static_cast( + WEBRTC_SPL_ABS_W16((distortion_lag - correlation_lag)) + 1); + RTC_DCHECK_LE(correlation_lags, static_cast(99 * fs_mult + 1)); + + for (size_t channel_ix = 0; channel_ix < num_channels_; ++channel_ix) { + ChannelParameters& parameters = channel_parameters_[channel_ix]; + if (channel_ix > 0) { + // When channel_ix == 0, audio_history contains the correct audio. For the + // other cases, we will have to copy the correct channel into + // audio_history. + (*sync_buffer_)[channel_ix].CopyTo(signal_length, audio_history_position, + audio_history.get()); + } + + // Calculate suitable scaling. + int16_t signal_max = WebRtcSpl_MaxAbsValueW16( + &audio_history[signal_length - correlation_length - start_index - + correlation_lags], + correlation_length + start_index + correlation_lags - 1); + int correlation_scale = + (31 - WebRtcSpl_NormW32(signal_max * signal_max)) + + (31 - WebRtcSpl_NormW32(static_cast(correlation_length))) - 31; + correlation_scale = std::max(0, correlation_scale); + + // Calculate the correlation, store in `correlation_vector2`. + WebRtcSpl_CrossCorrelation( + correlation_vector2, + &(audio_history[signal_length - correlation_length]), + &(audio_history[signal_length - correlation_length - start_index]), + correlation_length, correlation_lags, correlation_scale, -1); + + // Find maximizing index. + best_index = WebRtcSpl_MaxIndexW32(correlation_vector2, correlation_lags); + int32_t max_correlation = correlation_vector2[best_index]; + // Compensate index with start offset. + best_index = best_index + start_index; + + // Calculate energies. + int32_t energy1 = WebRtcSpl_DotProductWithScale( + &(audio_history[signal_length - correlation_length]), + &(audio_history[signal_length - correlation_length]), + correlation_length, correlation_scale); + int32_t energy2 = WebRtcSpl_DotProductWithScale( + &(audio_history[signal_length - correlation_length - best_index]), + &(audio_history[signal_length - correlation_length - best_index]), + correlation_length, correlation_scale); + + // Calculate the correlation coefficient between the two portions of the + // signal. + int32_t corr_coefficient; + if ((energy1 > 0) && (energy2 > 0)) { + int energy1_scale = std::max(16 - WebRtcSpl_NormW32(energy1), 0); + int energy2_scale = std::max(16 - WebRtcSpl_NormW32(energy2), 0); + // Make sure total scaling is even (to simplify scale factor after sqrt). + if ((energy1_scale + energy2_scale) & 1) { + // If sum is odd, add 1 to make it even. + energy1_scale += 1; + } + int32_t scaled_energy1 = energy1 >> energy1_scale; + int32_t scaled_energy2 = energy2 >> energy2_scale; + int16_t sqrt_energy_product = static_cast( + WebRtcSpl_SqrtFloor(scaled_energy1 * scaled_energy2)); + // Calculate max_correlation / sqrt(energy1 * energy2) in Q14. + int cc_shift = 14 - (energy1_scale + energy2_scale) / 2; + max_correlation = WEBRTC_SPL_SHIFT_W32(max_correlation, cc_shift); + corr_coefficient = + WebRtcSpl_DivW32W16(max_correlation, sqrt_energy_product); + // Cap at 1.0 in Q14. + corr_coefficient = std::min(16384, corr_coefficient); + } else { + corr_coefficient = 0; + } + + // Extract the two vectors expand_vector0 and expand_vector1 from + // `audio_history`. + size_t expansion_length = max_lag_ + overlap_length_; + const int16_t* vector1 = &(audio_history[signal_length - expansion_length]); + const int16_t* vector2 = vector1 - distortion_lag; + // Normalize the second vector to the same energy as the first. + energy1 = WebRtcSpl_DotProductWithScale(vector1, vector1, expansion_length, + correlation_scale); + energy2 = WebRtcSpl_DotProductWithScale(vector2, vector2, expansion_length, + correlation_scale); + // Confirm that amplitude ratio sqrt(energy1 / energy2) is within 0.5 - 2.0, + // i.e., energy1 / energy2 is within 0.25 - 4. + int16_t amplitude_ratio; + if ((energy1 / 4 < energy2) && (energy1 > energy2 / 4)) { + // Energy constraint fulfilled. Use both vectors and scale them + // accordingly. + int32_t scaled_energy2 = std::max(16 - WebRtcSpl_NormW32(energy2), 0); + int32_t scaled_energy1 = scaled_energy2 - 13; + // Calculate scaled_energy1 / scaled_energy2 in Q13. + int32_t energy_ratio = + WebRtcSpl_DivW32W16(WEBRTC_SPL_SHIFT_W32(energy1, -scaled_energy1), + static_cast(energy2 >> scaled_energy2)); + // Calculate sqrt ratio in Q13 (sqrt of en1/en2 in Q26). + amplitude_ratio = + static_cast(WebRtcSpl_SqrtFloor(energy_ratio << 13)); + // Copy the two vectors and give them the same energy. + parameters.expand_vector0.Clear(); + parameters.expand_vector0.PushBack(vector1, expansion_length); + parameters.expand_vector1.Clear(); + if (parameters.expand_vector1.Size() < expansion_length) { + parameters.expand_vector1.Extend(expansion_length - + parameters.expand_vector1.Size()); + } + std::unique_ptr temp_1(new int16_t[expansion_length]); + WebRtcSpl_AffineTransformVector( + temp_1.get(), const_cast(vector2), amplitude_ratio, 4096, + 13, expansion_length); + parameters.expand_vector1.OverwriteAt(temp_1.get(), expansion_length, 0); + } else { + // Energy change constraint not fulfilled. Only use last vector. + parameters.expand_vector0.Clear(); + parameters.expand_vector0.PushBack(vector1, expansion_length); + // Copy from expand_vector0 to expand_vector1. + parameters.expand_vector0.CopyTo(¶meters.expand_vector1); + // Set the energy_ratio since it is used by muting slope. + if ((energy1 / 4 < energy2) || (energy2 == 0)) { + amplitude_ratio = 4096; // 0.5 in Q13. + } else { + amplitude_ratio = 16384; // 2.0 in Q13. + } + } + + // Set the 3 lag values. + if (distortion_lag == correlation_lag) { + expand_lags_[0] = distortion_lag; + expand_lags_[1] = distortion_lag; + expand_lags_[2] = distortion_lag; + } else { + // `distortion_lag` and `correlation_lag` are not equal; use different + // combinations of the two. + // First lag is `distortion_lag` only. + expand_lags_[0] = distortion_lag; + // Second lag is the average of the two. + expand_lags_[1] = (distortion_lag + correlation_lag) / 2; + // Third lag is the average again, but rounding towards `correlation_lag`. + if (distortion_lag > correlation_lag) { + expand_lags_[2] = (distortion_lag + correlation_lag - 1) / 2; + } else { + expand_lags_[2] = (distortion_lag + correlation_lag + 1) / 2; + } + } + + // Calculate the LPC and the gain of the filters. + + // Calculate kUnvoicedLpcOrder + 1 lags of the auto-correlation function. + size_t temp_index = + signal_length - fs_mult_lpc_analysis_len - kUnvoicedLpcOrder; + // Copy signal to temporary vector to be able to pad with leading zeros. + int16_t* temp_signal = + new int16_t[fs_mult_lpc_analysis_len + kUnvoicedLpcOrder]; + memset(temp_signal, 0, + sizeof(int16_t) * (fs_mult_lpc_analysis_len + kUnvoicedLpcOrder)); + memcpy(&temp_signal[kUnvoicedLpcOrder], + &audio_history[temp_index + kUnvoicedLpcOrder], + sizeof(int16_t) * fs_mult_lpc_analysis_len); + CrossCorrelationWithAutoShift( + &temp_signal[kUnvoicedLpcOrder], &temp_signal[kUnvoicedLpcOrder], + fs_mult_lpc_analysis_len, kUnvoicedLpcOrder + 1, -1, auto_correlation); + delete[] temp_signal; + + // Verify that variance is positive. + if (auto_correlation[0] > 0) { + // Estimate AR filter parameters using Levinson-Durbin algorithm; + // kUnvoicedLpcOrder + 1 filter coefficients. + int16_t stability = + WebRtcSpl_LevinsonDurbin(auto_correlation, parameters.ar_filter, + reflection_coeff, kUnvoicedLpcOrder); + + // Keep filter parameters only if filter is stable. + if (stability != 1) { + // Set first coefficient to 4096 (1.0 in Q12). + parameters.ar_filter[0] = 4096; + // Set remaining `kUnvoicedLpcOrder` coefficients to zero. + WebRtcSpl_MemSetW16(parameters.ar_filter + 1, 0, kUnvoicedLpcOrder); + } + } + + if (channel_ix == 0) { + // Extract a noise segment. + size_t noise_length; + if (distortion_lag < 40) { + noise_length = 2 * distortion_lag + 30; + } else { + noise_length = distortion_lag + 30; + } + if (noise_length <= RandomVector::kRandomTableSize) { + memcpy(random_vector, RandomVector::kRandomTable, + sizeof(int16_t) * noise_length); + } else { + // Only applies to SWB where length could be larger than + // `kRandomTableSize`. + memcpy(random_vector, RandomVector::kRandomTable, + sizeof(int16_t) * RandomVector::kRandomTableSize); + RTC_DCHECK_LE(noise_length, kMaxSampleRate / 8000 * 120 + 30); + random_vector_->IncreaseSeedIncrement(2); + random_vector_->Generate( + noise_length - RandomVector::kRandomTableSize, + &random_vector[RandomVector::kRandomTableSize]); + } + } + + // Set up state vector and calculate scale factor for unvoiced filtering. + memcpy(parameters.ar_filter_state, + &(audio_history[signal_length - kUnvoicedLpcOrder]), + sizeof(int16_t) * kUnvoicedLpcOrder); + memcpy(unvoiced_vector - kUnvoicedLpcOrder, + &(audio_history[signal_length - 128 - kUnvoicedLpcOrder]), + sizeof(int16_t) * kUnvoicedLpcOrder); + WebRtcSpl_FilterMAFastQ12(&audio_history[signal_length - 128], + unvoiced_vector, parameters.ar_filter, + kUnvoicedLpcOrder + 1, 128); + const int unvoiced_max_abs = [&] { + const int16_t max_abs = WebRtcSpl_MaxAbsValueW16(unvoiced_vector, 128); + // Since WebRtcSpl_MaxAbsValueW16 returns 2^15 - 1 when the input contains + // -2^15, we have to conservatively bump the return value by 1 + // if it is 2^15 - 1. + return max_abs == WEBRTC_SPL_WORD16_MAX ? max_abs + 1 : max_abs; + }(); + // Pick the smallest n such that 2^n > unvoiced_max_abs; then the maximum + // value of the dot product is less than 2^7 * 2^(2*n) = 2^(2*n + 7), so to + // prevent overflows we want 2n + 7 <= 31, which means we should shift by + // 2n + 7 - 31 bits, if this value is greater than zero. + int unvoiced_prescale = + std::max(0, 2 * WebRtcSpl_GetSizeInBits(unvoiced_max_abs) - 24); + + int32_t unvoiced_energy = WebRtcSpl_DotProductWithScale( + unvoiced_vector, unvoiced_vector, 128, unvoiced_prescale); + + // Normalize `unvoiced_energy` to 28 or 29 bits to preserve sqrt() accuracy. + int16_t unvoiced_scale = WebRtcSpl_NormW32(unvoiced_energy) - 3; + // Make sure we do an odd number of shifts since we already have 7 shifts + // from dividing with 128 earlier. This will make the total scale factor + // even, which is suitable for the sqrt. + unvoiced_scale += ((unvoiced_scale & 0x1) ^ 0x1); + unvoiced_energy = WEBRTC_SPL_SHIFT_W32(unvoiced_energy, unvoiced_scale); + int16_t unvoiced_gain = + static_cast(WebRtcSpl_SqrtFloor(unvoiced_energy)); + parameters.ar_gain_scale = + 13 + (unvoiced_scale + 7 - unvoiced_prescale) / 2; + parameters.ar_gain = unvoiced_gain; + + // Calculate voice_mix_factor from corr_coefficient. + // Let x = corr_coefficient. Then, we compute: + // if (x > 0.48) + // voice_mix_factor = (-5179 + 19931x - 16422x^2 + 5776x^3) / 4096; + // else + // voice_mix_factor = 0; + if (corr_coefficient > 7875) { + int16_t x1, x2, x3; + // `corr_coefficient` is in Q14. + x1 = static_cast(corr_coefficient); + x2 = (x1 * x1) >> 14; // Shift 14 to keep result in Q14. + x3 = (x1 * x2) >> 14; + static const int kCoefficients[4] = {-5179, 19931, -16422, 5776}; + int32_t temp_sum = kCoefficients[0] * 16384; + temp_sum += kCoefficients[1] * x1; + temp_sum += kCoefficients[2] * x2; + temp_sum += kCoefficients[3] * x3; + parameters.voice_mix_factor = + static_cast(std::min(temp_sum / 4096, 16384)); + parameters.voice_mix_factor = + std::max(parameters.voice_mix_factor, static_cast(0)); + } else { + parameters.voice_mix_factor = 0; + } + + // Calculate muting slope. Reuse value from earlier scaling of + // `expand_vector0` and `expand_vector1`. + int16_t slope = amplitude_ratio; + if (slope > 12288) { + // slope > 1.5. + // Calculate (1 - (1 / slope)) / distortion_lag = + // (slope - 1) / (distortion_lag * slope). + // `slope` is in Q13, so 1 corresponds to 8192. Shift up to Q25 before + // the division. + // Shift the denominator from Q13 to Q5 before the division. The result of + // the division will then be in Q20. + int16_t denom = + rtc::saturated_cast((distortion_lag * slope) >> 8); + int temp_ratio = WebRtcSpl_DivW32W16((slope - 8192) << 12, denom); + if (slope > 14746) { + // slope > 1.8. + // Divide by 2, with proper rounding. + parameters.mute_slope = (temp_ratio + 1) / 2; + } else { + // Divide by 8, with proper rounding. + parameters.mute_slope = (temp_ratio + 4) / 8; + } + parameters.onset = true; + } else { + // Calculate (1 - slope) / distortion_lag. + // Shift `slope` by 7 to Q20 before the division. The result is in Q20. + parameters.mute_slope = WebRtcSpl_DivW32W16( + (8192 - slope) * 128, static_cast(distortion_lag)); + if (parameters.voice_mix_factor <= 13107) { + // Make sure the mute factor decreases from 1.0 to 0.9 in no more than + // 6.25 ms. + // mute_slope >= 0.005 / fs_mult in Q20. + parameters.mute_slope = std::max(5243 / fs_mult, parameters.mute_slope); + } else if (slope > 8028) { + parameters.mute_slope = 0; + } + parameters.onset = false; + } + } +} + +Expand::ChannelParameters::ChannelParameters() + : mute_factor(16384), + ar_gain(0), + ar_gain_scale(0), + voice_mix_factor(0), + current_voice_mix_factor(0), + onset(false), + mute_slope(0) { + memset(ar_filter, 0, sizeof(ar_filter)); + memset(ar_filter_state, 0, sizeof(ar_filter_state)); +} + +void Expand::Correlation(const int16_t* input, + size_t input_length, + int16_t* output) const { + // Set parameters depending on sample rate. + const int16_t* filter_coefficients; + size_t num_coefficients; + int16_t downsampling_factor; + if (fs_hz_ == 8000) { + num_coefficients = 3; + downsampling_factor = 2; + filter_coefficients = DspHelper::kDownsample8kHzTbl; + } else if (fs_hz_ == 16000) { + num_coefficients = 5; + downsampling_factor = 4; + filter_coefficients = DspHelper::kDownsample16kHzTbl; + } else if (fs_hz_ == 32000) { + num_coefficients = 7; + downsampling_factor = 8; + filter_coefficients = DspHelper::kDownsample32kHzTbl; + } else { // fs_hz_ == 48000. + num_coefficients = 7; + downsampling_factor = 12; + filter_coefficients = DspHelper::kDownsample48kHzTbl; + } + + // Correlate from lag 10 to lag 60 in downsampled domain. + // (Corresponds to 20-120 for narrow-band, 40-240 for wide-band, and so on.) + static const size_t kCorrelationStartLag = 10; + static const size_t kNumCorrelationLags = 54; + static const size_t kCorrelationLength = 60; + // Downsample to 4 kHz sample rate. + static const size_t kDownsampledLength = + kCorrelationStartLag + kNumCorrelationLags + kCorrelationLength; + int16_t downsampled_input[kDownsampledLength]; + static const size_t kFilterDelay = 0; + WebRtcSpl_DownsampleFast( + input + input_length - kDownsampledLength * downsampling_factor, + kDownsampledLength * downsampling_factor, downsampled_input, + kDownsampledLength, filter_coefficients, num_coefficients, + downsampling_factor, kFilterDelay); + + // Normalize `downsampled_input` to using all 16 bits. + int16_t max_value = + WebRtcSpl_MaxAbsValueW16(downsampled_input, kDownsampledLength); + int16_t norm_shift = 16 - WebRtcSpl_NormW32(max_value); + WebRtcSpl_VectorBitShiftW16(downsampled_input, kDownsampledLength, + downsampled_input, norm_shift); + + int32_t correlation[kNumCorrelationLags]; + CrossCorrelationWithAutoShift( + &downsampled_input[kDownsampledLength - kCorrelationLength], + &downsampled_input[kDownsampledLength - kCorrelationLength - + kCorrelationStartLag], + kCorrelationLength, kNumCorrelationLags, -1, correlation); + + // Normalize and move data from 32-bit to 16-bit vector. + int32_t max_correlation = + WebRtcSpl_MaxAbsValueW32(correlation, kNumCorrelationLags); + int16_t norm_shift2 = static_cast( + std::max(18 - WebRtcSpl_NormW32(max_correlation), 0)); + WebRtcSpl_VectorBitShiftW32ToW16(output, kNumCorrelationLags, correlation, + norm_shift2); +} + +void Expand::UpdateLagIndex() { + current_lag_index_ = current_lag_index_ + lag_index_direction_; + // Change direction if needed. + if (current_lag_index_ <= 0) { + lag_index_direction_ = 1; + } + if (current_lag_index_ >= kNumLags - 1) { + lag_index_direction_ = -1; + } +} + +Expand* ExpandFactory::Create(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels) const { + return new Expand(background_noise, sync_buffer, random_vector, statistics, + fs, num_channels); +} + +void Expand::GenerateRandomVector(int16_t seed_increment, + size_t length, + int16_t* random_vector) { + // TODO(turajs): According to hlundin The loop should not be needed. Should be + // just as good to generate all of the vector in one call. + size_t samples_generated = 0; + const size_t kMaxRandSamples = RandomVector::kRandomTableSize; + while (samples_generated < length) { + size_t rand_length = std::min(length - samples_generated, kMaxRandSamples); + random_vector_->IncreaseSeedIncrement(seed_increment); + random_vector_->Generate(rand_length, &random_vector[samples_generated]); + samples_generated += rand_length; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/expand.h b/third_party/libwebrtc/modules/audio_coding/neteq/expand.h new file mode 100644 index 0000000000..2e64583ec2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/expand.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ +#define MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ + + +#include + +#include "modules/audio_coding/neteq/audio_vector.h" + +namespace webrtc { + +// Forward declarations. +class AudioMultiVector; +class BackgroundNoise; +class RandomVector; +class StatisticsCalculator; +class SyncBuffer; + +// This class handles extrapolation of audio data from the sync_buffer to +// produce packet-loss concealment. +// TODO(hlundin): Refactor this class to divide the long methods into shorter +// ones. +class Expand { + public: + Expand(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels); + + virtual ~Expand(); + + Expand(const Expand&) = delete; + Expand& operator=(const Expand&) = delete; + + // Resets the object. + virtual void Reset(); + + // The main method to produce concealment data. The data is appended to the + // end of `output`. + virtual int Process(AudioMultiVector* output); + + // Prepare the object to do extra expansion during normal operation following + // a period of expands. + virtual void SetParametersForNormalAfterExpand(); + + // Prepare the object to do extra expansion during merge operation following + // a period of expands. + virtual void SetParametersForMergeAfterExpand(); + + // Returns the mute factor for `channel`. + int16_t MuteFactor(size_t channel) const { + RTC_DCHECK_LT(channel, num_channels_); + return channel_parameters_[channel].mute_factor; + } + + // Returns true if expansion has been faded down to zero amplitude (for all + // channels); false otherwise. + bool Muted() const; + + // Accessors and mutators. + virtual size_t overlap_length() const; + size_t max_lag() const { return max_lag_; } + + protected: + static const int kMaxConsecutiveExpands = 200; + void GenerateRandomVector(int16_t seed_increment, + size_t length, + int16_t* random_vector); + + // Initializes member variables at the beginning of an expand period. + void InitializeForAnExpandPeriod(); + + bool TooManyExpands(); + + // Analyzes the signal history in `sync_buffer_`, and set up all parameters + // necessary to produce concealment data. + void AnalyzeSignal(int16_t* random_vector); + + RandomVector* const random_vector_; + SyncBuffer* const sync_buffer_; + bool first_expand_; + const int fs_hz_; + const size_t num_channels_; + int consecutive_expands_; + + private: + static const size_t kUnvoicedLpcOrder = 6; + static const size_t kNumCorrelationCandidates = 3; + static const size_t kDistortionLength = 20; + static const size_t kLpcAnalysisLength = 160; + static const size_t kMaxSampleRate = 48000; + static const int kNumLags = 3; + + struct ChannelParameters { + ChannelParameters(); + int16_t mute_factor; + int16_t ar_filter[kUnvoicedLpcOrder + 1]; + int16_t ar_filter_state[kUnvoicedLpcOrder]; + int16_t ar_gain; + int16_t ar_gain_scale; + int16_t voice_mix_factor; /* Q14 */ + int16_t current_voice_mix_factor; /* Q14 */ + AudioVector expand_vector0; + AudioVector expand_vector1; + bool onset; + int mute_slope; /* Q20 */ + }; + + // Calculate the auto-correlation of `input`, with length `input_length` + // samples. The correlation is calculated from a downsampled version of + // `input`, and is written to `output`. + void Correlation(const int16_t* input, + size_t input_length, + int16_t* output) const; + + void UpdateLagIndex(); + + BackgroundNoise* const background_noise_; + StatisticsCalculator* const statistics_; + const size_t overlap_length_; + size_t max_lag_; + size_t expand_lags_[kNumLags]; + int lag_index_direction_; + int current_lag_index_; + bool stop_muting_; + size_t expand_duration_samples_; + std::unique_ptr channel_parameters_; +}; + +struct ExpandFactory { + ExpandFactory() {} + virtual ~ExpandFactory() {} + + virtual Expand* Create(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels) const; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.cc b/third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.cc new file mode 100644 index 0000000000..a91358b489 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.cc @@ -0,0 +1,71 @@ +/* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/expand_uma_logger.h" + +#include "absl/strings/string_view.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { +std::unique_ptr GetNewCountdown( + const TickTimer& tick_timer, + int logging_period_s) { + return tick_timer.GetNewCountdown((logging_period_s * 1000) / + tick_timer.ms_per_tick()); +} +} // namespace + +ExpandUmaLogger::ExpandUmaLogger(absl::string_view uma_name, + int logging_period_s, + const TickTimer* tick_timer) + : uma_name_(uma_name), + logging_period_s_(logging_period_s), + tick_timer_(*tick_timer), + timer_(GetNewCountdown(tick_timer_, logging_period_s_)) { + RTC_DCHECK(tick_timer); + RTC_DCHECK_GT(logging_period_s_, 0); +} + +ExpandUmaLogger::~ExpandUmaLogger() = default; + +void ExpandUmaLogger::UpdateSampleCounter(uint64_t samples, + int sample_rate_hz) { + if ((last_logged_value_ && *last_logged_value_ > samples) || + sample_rate_hz_ != sample_rate_hz) { + // Sanity checks. The incremental counter moved backwards, or sample rate + // changed. + last_logged_value_.reset(); + } + last_value_ = samples; + sample_rate_hz_ = sample_rate_hz; + if (!last_logged_value_) { + last_logged_value_ = absl::optional(samples); + } + + if (!timer_->Finished()) { + // Not yet time to log. + return; + } + + RTC_DCHECK(last_logged_value_); + RTC_DCHECK_GE(last_value_, *last_logged_value_); + const uint64_t diff = last_value_ - *last_logged_value_; + last_logged_value_ = absl::optional(last_value_); + // Calculate rate in percent. + RTC_DCHECK_GT(sample_rate_hz, 0); + const int rate = (100 * diff) / (sample_rate_hz * logging_period_s_); + RTC_DCHECK_GE(rate, 0); + RTC_DCHECK_LE(rate, 100); + RTC_HISTOGRAM_PERCENTAGE_SPARSE(uma_name_, rate); + timer_ = GetNewCountdown(tick_timer_, logging_period_s_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.h b/third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.h new file mode 100644 index 0000000000..cc5c20a886 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_EXPAND_UMA_LOGGER_H_ +#define MODULES_AUDIO_CODING_NETEQ_EXPAND_UMA_LOGGER_H_ + +#include + +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/neteq/tick_timer.h" + +namespace webrtc { + +// This class is used to periodically log values to a UMA histogram. The caller +// is expected to update this class with an incremental sample counter which +// counts expand samples. At the end of each logging period, the class will +// calculate the fraction of samples that were expand samples during that period +// and report that in percent. The logging period must be strictly positive. +// Does not take ownership of tick_timer and the pointer must refer to a valid +// object that outlives the one constructed. +class ExpandUmaLogger { + public: + ExpandUmaLogger(absl::string_view uma_name, + int logging_period_s, + const TickTimer* tick_timer); + + ~ExpandUmaLogger(); + + ExpandUmaLogger(const ExpandUmaLogger&) = delete; + ExpandUmaLogger& operator=(const ExpandUmaLogger&) = delete; + + // In this call, value should be an incremental sample counter. The sample + // rate must be strictly positive. + void UpdateSampleCounter(uint64_t value, int sample_rate_hz); + + private: + const std::string uma_name_; + const int logging_period_s_; + const TickTimer& tick_timer_; + std::unique_ptr timer_; + absl::optional last_logged_value_; + uint64_t last_value_ = 0; + int sample_rate_hz_ = 0; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_EXPAND_UMA_LOGGER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/expand_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/expand_unittest.cc new file mode 100644 index 0000000000..9355fce5e1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/expand_unittest.cc @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for Expand class. + +#include "modules/audio_coding/neteq/expand.h" + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +TEST(Expand, CreateAndDestroy) { + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); +} + +TEST(Expand, CreateUsingFactory) { + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + ExpandFactory expand_factory; + Expand* expand = expand_factory.Create(&bgn, &sync_buffer, &random_vector, + &statistics, fs, channels); + EXPECT_TRUE(expand != NULL); + delete expand; +} + +namespace { +class FakeStatisticsCalculator : public StatisticsCalculator { + public: + void LogDelayedPacketOutageEvent(int num_samples, int fs_hz) override { + last_outage_duration_samples_ = num_samples; + } + + int last_outage_duration_samples() const { + return last_outage_duration_samples_; + } + + private: + int last_outage_duration_samples_ = 0; +}; + +// This is the same size that is given to the SyncBuffer object in NetEq. +const size_t kNetEqSyncBufferLengthMs = 720; +} // namespace + +class ExpandTest : public ::testing::Test { + protected: + ExpandTest() + : input_file_(test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000), + test_sample_rate_hz_(32000), + num_channels_(1), + background_noise_(num_channels_), + sync_buffer_(num_channels_, + kNetEqSyncBufferLengthMs * test_sample_rate_hz_ / 1000), + expand_(&background_noise_, + &sync_buffer_, + &random_vector_, + &statistics_, + test_sample_rate_hz_, + num_channels_) { + input_file_.set_output_rate_hz(test_sample_rate_hz_); + } + + void SetUp() override { + // Fast-forward the input file until there is speech (about 1.1 second into + // the file). + const int speech_start_samples = + static_cast(test_sample_rate_hz_ * 1.1f); + ASSERT_TRUE(input_file_.Seek(speech_start_samples)); + + // Pre-load the sync buffer with speech data. + std::unique_ptr temp(new int16_t[sync_buffer_.Size()]); + ASSERT_TRUE(input_file_.Read(sync_buffer_.Size(), temp.get())); + sync_buffer_.Channel(0).OverwriteAt(temp.get(), sync_buffer_.Size(), 0); + ASSERT_EQ(1u, num_channels_) << "Fix: Must populate all channels."; + } + + test::ResampleInputAudioFile input_file_; + int test_sample_rate_hz_; + size_t num_channels_; + BackgroundNoise background_noise_; + SyncBuffer sync_buffer_; + RandomVector random_vector_; + FakeStatisticsCalculator statistics_; + Expand expand_; +}; + +// This test calls the expand object to produce concealment data a few times, +// and then ends by calling SetParametersForNormalAfterExpand. This simulates +// the situation where the packet next up for decoding was just delayed, not +// lost. +TEST_F(ExpandTest, DelayedPacketOutage) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + EXPECT_EQ(0, statistics_.last_outage_duration_samples()); + } + expand_.SetParametersForNormalAfterExpand(); + // Convert `sum_output_len_samples` to milliseconds. + EXPECT_EQ(rtc::checked_cast(sum_output_len_samples), + statistics_.last_outage_duration_samples()); +} + +// This test is similar to DelayedPacketOutage, but ends by calling +// SetParametersForMergeAfterExpand. This simulates the situation where the +// packet next up for decoding was actually lost (or at least a later packet +// arrived before it). +TEST_F(ExpandTest, LostPacketOutage) { + AudioMultiVector output(num_channels_); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + EXPECT_EQ(0, statistics_.last_outage_duration_samples()); + } + expand_.SetParametersForMergeAfterExpand(); + EXPECT_EQ(0, statistics_.last_outage_duration_samples()); +} + +// This test is similar to the DelayedPacketOutage test above, but with the +// difference that Expand::Reset() is called after 5 calls to Expand::Process(). +// This should reset the statistics, and will in the end lead to an outage of +// 5 periods instead of 10. +TEST_F(ExpandTest, CheckOutageStatsAfterReset) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + if (i == 5) { + expand_.Reset(); + sum_output_len_samples = 0; + } + EXPECT_EQ(0, statistics_.last_outage_duration_samples()); + } + expand_.SetParametersForNormalAfterExpand(); + // Convert `sum_output_len_samples` to milliseconds. + EXPECT_EQ(rtc::checked_cast(sum_output_len_samples), + statistics_.last_outage_duration_samples()); +} + +namespace { +// Runs expand until Muted() returns true. Times out after 1000 calls. +void ExpandUntilMuted(size_t num_channels, Expand* expand) { + EXPECT_FALSE(expand->Muted()) << "Instance is muted from the start"; + AudioMultiVector output(num_channels); + int num_calls = 0; + while (!expand->Muted()) { + ASSERT_LT(num_calls++, 1000) << "Test timed out"; + EXPECT_EQ(0, expand->Process(&output)); + } +} +} // namespace + +// Verifies that Muted() returns true after a long expand period. Also verifies +// that Muted() is reset to false after calling Reset(), +// SetParametersForMergeAfterExpand() and SetParametersForNormalAfterExpand(). +TEST_F(ExpandTest, Muted) { + ExpandUntilMuted(num_channels_, &expand_); + expand_.Reset(); + EXPECT_FALSE(expand_.Muted()); // Should be back to unmuted. + + ExpandUntilMuted(num_channels_, &expand_); + expand_.SetParametersForMergeAfterExpand(); + EXPECT_FALSE(expand_.Muted()); // Should be back to unmuted. + + expand_.Reset(); // Must reset in order to start a new expand period. + ExpandUntilMuted(num_channels_, &expand_); + expand_.SetParametersForNormalAfterExpand(); + EXPECT_FALSE(expand_.Muted()); // Should be back to unmuted. +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/g3doc/index.md b/third_party/libwebrtc/modules/audio_coding/neteq/g3doc/index.md new file mode 100644 index 0000000000..40e76e2742 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/g3doc/index.md @@ -0,0 +1,102 @@ + + + +# NetEq + +NetEq is the audio jitter buffer and packet loss concealer. The jitter buffer is +an adaptive jitter buffer, meaning that the buffering delay is continuously +optimized based on the network conditions. Its main goal is to ensure a smooth +playout of incoming audio packets from the network with a low amount of audio +artifacts (alterations to the original content of the packets) while at the same +time keep the delay as low as possible. + +## API + +At a high level, the NetEq API has two main functions: +[`InsertPacket`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/neteq/neteq.h;l=198;drc=4461f059d180fe8c2886d422ebd1cb55b5c83e72) +and +[`GetAudio`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/neteq/neteq.h;l=219;drc=4461f059d180fe8c2886d422ebd1cb55b5c83e72). + +### InsertPacket + +[`InsertPacket`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/neteq/neteq.h;l=198;drc=4461f059d180fe8c2886d422ebd1cb55b5c83e72) +delivers an RTP packet from the network to NetEq where the following happens: + +1. The packet is discarded if it is too late for playout (for example if it was + reordered). Otherwize it is put into the packet buffer where it is stored + until it is time for playout. If the buffer is full, discard all the + existing packets (this should be rare). +2. The interarrival time between packets is analyzed and statistics is updated + which is used to derive a new target playout delay. The interarrival time is + measured in the number of GetAudio ‘ticks’ and thus clock drift between the + sender and receiver can be accounted for. + +### GetAudio + +[`GetAudio`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/neteq/neteq.h;l=219;drc=4461f059d180fe8c2886d422ebd1cb55b5c83e72) +pulls 10 ms of audio from NetEq for playout. A much simplified decision logic is +as follows: + +1. If there is 10 ms audio in the sync buffer then return that. +2. If the next packet is available (based on RTP timestamp) in the packet + buffer then decode it and append the result to the sync buffer. + 1. Compare the current delay estimate (filtered buffer level) with the + target delay and time stretch (accelerate or decelerate) the contents of + the sync buffer if the buffer level is too high or too low. + 2. Return 10 ms of audio from the sync buffer. +3. If the last decoded packet was a discontinuous transmission (DTX) packet + then generate comfort noise. +4. If there is no available packet for decoding due to the next packet having + not arrived or been lost then generate packet loss concealment by + extrapolating the remaining audio in the sync buffer or by asking the + decoder to produce it. + +In summary, the output is the result one of the following operations: + +* Normal: audio decoded from a packet. +* Acceleration: accelerated playout of a decoded packet. +* Preemptive expand: decelerated playout of a decoded packet. +* Expand: packet loss concealment generated by NetEq or the decoder. +* Merge: audio stitched together from packet loss concealment to decoded data + in case of a loss. +* Comfort noise (CNG): comfort noise generated by NetEq or the decoder between + talk spurts due to discontinuous transmission of packets (DTX). + +## Statistics + +There are a number of functions that can be used to query the internal state of +NetEq, statistics about the type of audio output and latency metrics such as how +long time packets have waited in the buffer. + +* [`NetworkStatistics`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/neteq/neteq.h;l=273;drc=4461f059d180fe8c2886d422ebd1cb55b5c83e72): + instantaneous values or stats averaged over the duration since last call to + this function. +* [`GetLifetimeStatistics`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/neteq/neteq.h;l=280;drc=4461f059d180fe8c2886d422ebd1cb55b5c83e72): + cumulative stats that persist over the lifetime of the class. +* [`GetOperationsAndState`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/neteq/neteq.h;l=284;drc=4461f059d180fe8c2886d422ebd1cb55b5c83e72): + information about the internal state of NetEq (is only inteded to be used + for testing and debugging). + +## Tests and tools + +* [`neteq_rtpplay`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc;drc=cee751abff598fc19506f77de08bea7c61b9dcca): + Simulate NetEq behavior based on either an RTP dump, a PCAP file or an RTC + event log. A replacement audio file can also be used instead of the original + payload. Outputs aggregated statistics and optionally an audio file to + listen to. +* [`neteq_speed_test`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc;drc=2ab97f6f8e27b47c0d9beeb8b6ca5387bda9f55c): + Measure performance of NetEq, used on perf bots. +* Unit tests including bit exactness tests where RTP file is used as an input + to NetEq, the output is concatenated and a checksum is calculated and + compared against a reference. + +## Other responsibilities + +* Dual-tone multi-frequency signaling (DTMF): receive telephone events and + produce dual tone waveforms. +* Forward error correction (RED or codec inband FEC): split inserted packets + and prioritize the payloads. +* NACK (negative acknowledgement): keep track of lost packets and generate a + list of packets to NACK. +* Audio/video sync: NetEq can be instructed to increase the latency in order + to keep audio and video in sync. diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/histogram.cc b/third_party/libwebrtc/modules/audio_coding/neteq/histogram.cc new file mode 100644 index 0000000000..e4b7f10379 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/histogram.cc @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/histogram.h" + +#include +#include +#include + +#include "absl/types/optional.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +Histogram::Histogram(size_t num_buckets, + int forget_factor, + absl::optional start_forget_weight) + : buckets_(num_buckets, 0), + forget_factor_(0), + base_forget_factor_(forget_factor), + add_count_(0), + start_forget_weight_(start_forget_weight) { + RTC_DCHECK_LT(base_forget_factor_, 1 << 15); +} + +Histogram::~Histogram() {} + +// Each element in the vector is first multiplied by the forgetting factor +// `forget_factor_`. Then the vector element indicated by `iat_packets` is then +// increased (additive) by 1 - `forget_factor_`. This way, the probability of +// `value` is slightly increased, while the sum of the histogram remains +// constant (=1). +// Due to inaccuracies in the fixed-point arithmetic, the histogram may no +// longer sum up to 1 (in Q30) after the update. To correct this, a correction +// term is added or subtracted from the first element (or elements) of the +// vector. +// The forgetting factor `forget_factor_` is also updated. When the DelayManager +// is reset, the factor is set to 0 to facilitate rapid convergence in the +// beginning. With each update of the histogram, the factor is increased towards +// the steady-state value `base_forget_factor_`. +void Histogram::Add(int value) { + RTC_DCHECK(value >= 0); + RTC_DCHECK(value < static_cast(buckets_.size())); + int vector_sum = 0; // Sum up the vector elements as they are processed. + // Multiply each element in `buckets_` with `forget_factor_`. + for (int& bucket : buckets_) { + bucket = (static_cast(bucket) * forget_factor_) >> 15; + vector_sum += bucket; + } + + // Increase the probability for the currently observed inter-arrival time + // by 1 - `forget_factor_`. The factor is in Q15, `buckets_` in Q30. + // Thus, left-shift 15 steps to obtain result in Q30. + buckets_[value] += (32768 - forget_factor_) << 15; + vector_sum += (32768 - forget_factor_) << 15; // Add to vector sum. + + // `buckets_` should sum up to 1 (in Q30), but it may not due to + // fixed-point rounding errors. + vector_sum -= 1 << 30; // Should be zero. Compensate if not. + if (vector_sum != 0) { + // Modify a few values early in `buckets_`. + int flip_sign = vector_sum > 0 ? -1 : 1; + for (int& bucket : buckets_) { + // Add/subtract 1/16 of the element, but not more than `vector_sum`. + int correction = flip_sign * std::min(std::abs(vector_sum), bucket >> 4); + bucket += correction; + vector_sum += correction; + if (std::abs(vector_sum) == 0) { + break; + } + } + } + RTC_DCHECK(vector_sum == 0); // Verify that the above is correct. + + ++add_count_; + + // Update `forget_factor_` (changes only during the first seconds after a + // reset). The factor converges to `base_forget_factor_`. + if (start_forget_weight_) { + if (forget_factor_ != base_forget_factor_) { + int old_forget_factor = forget_factor_; + int forget_factor = + (1 << 15) * (1 - start_forget_weight_.value() / (add_count_ + 1)); + forget_factor_ = + std::max(0, std::min(base_forget_factor_, forget_factor)); + // The histogram is updated recursively by forgetting the old histogram + // with `forget_factor_` and adding a new sample multiplied by |1 - + // forget_factor_|. We need to make sure that the effective weight on the + // new sample is no smaller than those on the old samples, i.e., to + // satisfy the following DCHECK. + RTC_DCHECK_GE((1 << 15) - forget_factor_, + ((1 << 15) - old_forget_factor) * forget_factor_ >> 15); + } + } else { + forget_factor_ += (base_forget_factor_ - forget_factor_ + 3) >> 2; + } +} + +int Histogram::Quantile(int probability) { + // Find the bucket for which the probability of observing an + // inter-arrival time larger than or equal to `index` is larger than or + // equal to `probability`. The sought probability is estimated using + // the histogram as the reverse cumulant PDF, i.e., the sum of elements from + // the end up until `index`. Now, since the sum of all elements is 1 + // (in Q30) by definition, and since the solution is often a low value for + // `iat_index`, it is more efficient to start with `sum` = 1 and subtract + // elements from the start of the histogram. + int inverse_probability = (1 << 30) - probability; + size_t index = 0; // Start from the beginning of `buckets_`. + int sum = 1 << 30; // Assign to 1 in Q30. + sum -= buckets_[index]; + + while ((sum > inverse_probability) && (index < buckets_.size() - 1)) { + // Subtract the probabilities one by one until the sum is no longer greater + // than `inverse_probability`. + ++index; + sum -= buckets_[index]; + } + return static_cast(index); +} + +// Set the histogram vector to an exponentially decaying distribution +// buckets_[i] = 0.5^(i+1), i = 0, 1, 2, ... +// buckets_ is in Q30. +void Histogram::Reset() { + // Set temp_prob to (slightly more than) 1 in Q14. This ensures that the sum + // of buckets_ is 1. + uint16_t temp_prob = 0x4002; // 16384 + 2 = 100000000000010 binary. + for (int& bucket : buckets_) { + temp_prob >>= 1; + bucket = temp_prob << 16; + } + forget_factor_ = 0; // Adapt the histogram faster for the first few packets. + add_count_ = 0; +} + +int Histogram::NumBuckets() const { + return buckets_.size(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/histogram.h b/third_party/libwebrtc/modules/audio_coding/neteq/histogram.h new file mode 100644 index 0000000000..265a10e00a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/histogram.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_HISTOGRAM_H_ +#define MODULES_AUDIO_CODING_NETEQ_HISTOGRAM_H_ + +#include // Provide access to size_t. + +#include + +#include "absl/types/optional.h" + +namespace webrtc { + +class Histogram { + public: + // Creates histogram with capacity `num_buckets` and `forget_factor` in Q15. + Histogram(size_t num_buckets, + int forget_factor, + absl::optional start_forget_weight = absl::nullopt); + + virtual ~Histogram(); + + // Resets the histogram to the default start distribution. + virtual void Reset(); + + // Add entry in bucket `index`. + virtual void Add(int index); + + // Calculates the quantile at `probability` (in Q30) of the histogram + // distribution. + virtual int Quantile(int probability); + + // Returns the number of buckets in the histogram. + virtual int NumBuckets() const; + + // Returns the probability for each bucket in Q30. + const std::vector& buckets() const { return buckets_; } + + // Accessors only intended for testing purposes. + int base_forget_factor_for_testing() const { return base_forget_factor_; } + int forget_factor_for_testing() const { return forget_factor_; } + absl::optional start_forget_weight_for_testing() const { + return start_forget_weight_; + } + + private: + std::vector buckets_; + int forget_factor_; // Q15 + const int base_forget_factor_; + int add_count_; + const absl::optional start_forget_weight_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_HISTOGRAM_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/histogram_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/histogram_unittest.cc new file mode 100644 index 0000000000..e30a2956dc --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/histogram_unittest.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/histogram.h" + +#include + +#include "test/gtest.h" + +namespace webrtc { + +TEST(HistogramTest, Initialization) { + Histogram histogram(65, 32440); + histogram.Reset(); + const auto& buckets = histogram.buckets(); + double sum = 0.0; + for (size_t i = 0; i < buckets.size(); i++) { + EXPECT_NEAR(ldexp(std::pow(0.5, static_cast(i + 1)), 30), buckets[i], + 65537); + // Tolerance 65537 in Q30 corresponds to a delta of approximately 0.00006. + sum += buckets[i]; + } + EXPECT_EQ(1 << 30, static_cast(sum)); // Should be 1 in Q30. +} + +TEST(HistogramTest, Add) { + Histogram histogram(10, 32440); + histogram.Reset(); + const std::vector before = histogram.buckets(); + const int index = 5; + histogram.Add(index); + const std::vector after = histogram.buckets(); + EXPECT_GT(after[index], before[index]); + int sum = 0; + for (int bucket : after) { + sum += bucket; + } + EXPECT_EQ(1 << 30, sum); +} + +TEST(HistogramTest, ForgetFactor) { + Histogram histogram(10, 32440); + histogram.Reset(); + const std::vector before = histogram.buckets(); + const int index = 4; + histogram.Add(index); + const std::vector after = histogram.buckets(); + for (int i = 0; i < histogram.NumBuckets(); ++i) { + if (i != index) { + EXPECT_LT(after[i], before[i]); + } + } +} + +TEST(HistogramTest, ReachSteadyStateForgetFactor) { + static constexpr int kSteadyStateForgetFactor = (1 << 15) * 0.9993; + Histogram histogram(100, kSteadyStateForgetFactor, 1.0); + histogram.Reset(); + int n = (1 << 15) / ((1 << 15) - kSteadyStateForgetFactor); + for (int i = 0; i < n; ++i) { + histogram.Add(0); + } + EXPECT_EQ(histogram.forget_factor_for_testing(), kSteadyStateForgetFactor); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/merge.cc b/third_party/libwebrtc/modules/audio_coding/neteq/merge.cc new file mode 100644 index 0000000000..0aec6d2597 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/merge.cc @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/merge.h" + +#include // memmove, memcpy, memset, size_t + +#include // min, max +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/cross_correlation.h" +#include "modules/audio_coding/neteq/dsp_helper.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +Merge::Merge(int fs_hz, + size_t num_channels, + Expand* expand, + SyncBuffer* sync_buffer) + : fs_hz_(fs_hz), + num_channels_(num_channels), + fs_mult_(fs_hz_ / 8000), + timestamps_per_call_(static_cast(fs_hz_ / 100)), + expand_(expand), + sync_buffer_(sync_buffer), + expanded_(num_channels_) { + RTC_DCHECK_GT(num_channels_, 0); +} + +Merge::~Merge() = default; + +size_t Merge::Process(int16_t* input, + size_t input_length, + AudioMultiVector* output) { + // TODO(hlundin): Change to an enumerator and skip assert. + RTC_DCHECK(fs_hz_ == 8000 || fs_hz_ == 16000 || fs_hz_ == 32000 || + fs_hz_ == 48000); + RTC_DCHECK_LE(fs_hz_, kMaxSampleRate); // Should not be possible. + if (input_length == 0) { + return 0; + } + + size_t old_length; + size_t expand_period; + // Get expansion data to overlap and mix with. + size_t expanded_length = GetExpandedSignal(&old_length, &expand_period); + + // Transfer input signal to an AudioMultiVector. + AudioMultiVector input_vector(num_channels_); + input_vector.PushBackInterleaved( + rtc::ArrayView(input, input_length)); + size_t input_length_per_channel = input_vector.Size(); + RTC_DCHECK_EQ(input_length_per_channel, input_length / num_channels_); + + size_t best_correlation_index = 0; + size_t output_length = 0; + + std::unique_ptr input_channel( + new int16_t[input_length_per_channel]); + std::unique_ptr expanded_channel(new int16_t[expanded_length]); + for (size_t channel = 0; channel < num_channels_; ++channel) { + input_vector[channel].CopyTo(input_length_per_channel, 0, + input_channel.get()); + expanded_[channel].CopyTo(expanded_length, 0, expanded_channel.get()); + + const int16_t new_mute_factor = std::min( + 16384, SignalScaling(input_channel.get(), input_length_per_channel, + expanded_channel.get())); + + if (channel == 0) { + // Downsample, correlate, and find strongest correlation period for the + // reference (i.e., first) channel only. + // Downsample to 4kHz sample rate. + Downsample(input_channel.get(), input_length_per_channel, + expanded_channel.get(), expanded_length); + + // Calculate the lag of the strongest correlation period. + best_correlation_index = CorrelateAndPeakSearch( + old_length, input_length_per_channel, expand_period); + } + + temp_data_.resize(input_length_per_channel + best_correlation_index); + int16_t* decoded_output = temp_data_.data() + best_correlation_index; + + // Mute the new decoded data if needed (and unmute it linearly). + // This is the overlapping part of expanded_signal. + size_t interpolation_length = + std::min(kMaxCorrelationLength * fs_mult_, + expanded_length - best_correlation_index); + interpolation_length = + std::min(interpolation_length, input_length_per_channel); + + RTC_DCHECK_LE(new_mute_factor, 16384); + int16_t mute_factor = + std::max(expand_->MuteFactor(channel), new_mute_factor); + RTC_DCHECK_GE(mute_factor, 0); + + if (mute_factor < 16384) { + // Set a suitable muting slope (Q20). 0.004 for NB, 0.002 for WB, + // and so on, or as fast as it takes to come back to full gain within the + // frame length. + const int back_to_fullscale_inc = static_cast( + ((16384 - mute_factor) << 6) / input_length_per_channel); + const int increment = std::max(4194 / fs_mult_, back_to_fullscale_inc); + mute_factor = static_cast(DspHelper::RampSignal( + input_channel.get(), interpolation_length, mute_factor, increment)); + DspHelper::UnmuteSignal(&input_channel[interpolation_length], + input_length_per_channel - interpolation_length, + &mute_factor, increment, + &decoded_output[interpolation_length]); + } else { + // No muting needed. + memmove( + &decoded_output[interpolation_length], + &input_channel[interpolation_length], + sizeof(int16_t) * (input_length_per_channel - interpolation_length)); + } + + // Do overlap and mix linearly. + int16_t increment = + static_cast(16384 / (interpolation_length + 1)); // In Q14. + int16_t local_mute_factor = 16384 - increment; + memmove(temp_data_.data(), expanded_channel.get(), + sizeof(int16_t) * best_correlation_index); + DspHelper::CrossFade(&expanded_channel[best_correlation_index], + input_channel.get(), interpolation_length, + &local_mute_factor, increment, decoded_output); + + output_length = best_correlation_index + input_length_per_channel; + if (channel == 0) { + RTC_DCHECK(output->Empty()); // Output should be empty at this point. + output->AssertSize(output_length); + } else { + RTC_DCHECK_EQ(output->Size(), output_length); + } + (*output)[channel].OverwriteAt(temp_data_.data(), output_length, 0); + } + + // Copy back the first part of the data to `sync_buffer_` and remove it from + // `output`. + sync_buffer_->ReplaceAtIndex(*output, old_length, sync_buffer_->next_index()); + output->PopFront(old_length); + + // Return new added length. `old_length` samples were borrowed from + // `sync_buffer_`. + RTC_DCHECK_GE(output_length, old_length); + return output_length - old_length; +} + +size_t Merge::GetExpandedSignal(size_t* old_length, size_t* expand_period) { + // Check how much data that is left since earlier. + *old_length = sync_buffer_->FutureLength(); + // Should never be less than overlap_length. + RTC_DCHECK_GE(*old_length, expand_->overlap_length()); + // Generate data to merge the overlap with using expand. + expand_->SetParametersForMergeAfterExpand(); + + if (*old_length >= 210 * kMaxSampleRate / 8000) { + // TODO(hlundin): Write test case for this. + // The number of samples available in the sync buffer is more than what fits + // in expanded_signal. Keep the first 210 * kMaxSampleRate / 8000 samples, + // but shift them towards the end of the buffer. This is ok, since all of + // the buffer will be expand data anyway, so as long as the beginning is + // left untouched, we're fine. + size_t length_diff = *old_length - 210 * kMaxSampleRate / 8000; + sync_buffer_->InsertZerosAtIndex(length_diff, sync_buffer_->next_index()); + *old_length = 210 * kMaxSampleRate / 8000; + // This is the truncated length. + } + // This assert should always be true thanks to the if statement above. + RTC_DCHECK_GE(210 * kMaxSampleRate / 8000, *old_length); + + AudioMultiVector expanded_temp(num_channels_); + expand_->Process(&expanded_temp); + *expand_period = expanded_temp.Size(); // Samples per channel. + + expanded_.Clear(); + // Copy what is left since earlier into the expanded vector. + expanded_.PushBackFromIndex(*sync_buffer_, sync_buffer_->next_index()); + RTC_DCHECK_EQ(expanded_.Size(), *old_length); + RTC_DCHECK_GT(expanded_temp.Size(), 0); + // Do "ugly" copy and paste from the expanded in order to generate more data + // to correlate (but not interpolate) with. + const size_t required_length = static_cast((120 + 80 + 2) * fs_mult_); + if (expanded_.Size() < required_length) { + while (expanded_.Size() < required_length) { + // Append one more pitch period each time. + expanded_.PushBack(expanded_temp); + } + // Trim the length to exactly `required_length`. + expanded_.PopBack(expanded_.Size() - required_length); + } + RTC_DCHECK_GE(expanded_.Size(), required_length); + return required_length; +} + +int16_t Merge::SignalScaling(const int16_t* input, + size_t input_length, + const int16_t* expanded_signal) const { + // Adjust muting factor if new vector is more or less of the BGN energy. + const auto mod_input_length = rtc::SafeMin( + 64 * rtc::dchecked_cast(fs_mult_), input_length); + + // Missing input, do no muting + if (mod_input_length == 0) { + return 16384; + } + + const int16_t expanded_max = + WebRtcSpl_MaxAbsValueW16(expanded_signal, mod_input_length); + int32_t factor = + (expanded_max * expanded_max) / (std::numeric_limits::max() / + static_cast(mod_input_length)); + const int expanded_shift = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); + int32_t energy_expanded = WebRtcSpl_DotProductWithScale( + expanded_signal, expanded_signal, mod_input_length, expanded_shift); + + // Calculate energy of input signal. + const int16_t input_max = WebRtcSpl_MaxAbsValueW16(input, mod_input_length); + factor = (input_max * input_max) / (std::numeric_limits::max() / + static_cast(mod_input_length)); + const int input_shift = factor == 0 ? 0 : 31 - WebRtcSpl_NormW32(factor); + int32_t energy_input = WebRtcSpl_DotProductWithScale( + input, input, mod_input_length, input_shift); + + // Align to the same Q-domain. + if (input_shift > expanded_shift) { + energy_expanded = energy_expanded >> (input_shift - expanded_shift); + } else { + energy_input = energy_input >> (expanded_shift - input_shift); + } + + // Calculate muting factor to use for new frame. + int16_t mute_factor; + if (energy_input > energy_expanded) { + // Normalize `energy_input` to 14 bits. + int16_t temp_shift = WebRtcSpl_NormW32(energy_input) - 17; + energy_input = WEBRTC_SPL_SHIFT_W32(energy_input, temp_shift); + // Put `energy_expanded` in a domain 14 higher, so that + // energy_expanded / energy_input is in Q14. + energy_expanded = WEBRTC_SPL_SHIFT_W32(energy_expanded, temp_shift + 14); + // Calculate sqrt(energy_expanded / energy_input) in Q14. + mute_factor = static_cast( + WebRtcSpl_SqrtFloor((energy_expanded / energy_input) << 14)); + } else { + // Set to 1 (in Q14) when `expanded` has higher energy than `input`. + mute_factor = 16384; + } + + return mute_factor; +} + +// TODO(hlundin): There are some parameter values in this method that seem +// strange. Compare with Expand::Correlation. +void Merge::Downsample(const int16_t* input, + size_t input_length, + const int16_t* expanded_signal, + size_t expanded_length) { + const int16_t* filter_coefficients; + size_t num_coefficients; + int decimation_factor = fs_hz_ / 4000; + static const size_t kCompensateDelay = 0; + size_t length_limit = static_cast(fs_hz_ / 100); // 10 ms in samples. + if (fs_hz_ == 8000) { + filter_coefficients = DspHelper::kDownsample8kHzTbl; + num_coefficients = 3; + } else if (fs_hz_ == 16000) { + filter_coefficients = DspHelper::kDownsample16kHzTbl; + num_coefficients = 5; + } else if (fs_hz_ == 32000) { + filter_coefficients = DspHelper::kDownsample32kHzTbl; + num_coefficients = 7; + } else { // fs_hz_ == 48000 + filter_coefficients = DspHelper::kDownsample48kHzTbl; + num_coefficients = 7; + } + size_t signal_offset = num_coefficients - 1; + WebRtcSpl_DownsampleFast( + &expanded_signal[signal_offset], expanded_length - signal_offset, + expanded_downsampled_, kExpandDownsampLength, filter_coefficients, + num_coefficients, decimation_factor, kCompensateDelay); + if (input_length <= length_limit) { + // Not quite long enough, so we have to cheat a bit. + // If the input is shorter than the offset, we consider the input to be 0 + // length. This will cause us to skip the downsampling since it makes no + // sense anyway, and input_downsampled_ will be filled with zeros. This is + // clearly a pathological case, and the signal quality will suffer, but + // there is not much we can do. + const size_t temp_len = + input_length > signal_offset ? input_length - signal_offset : 0; + // TODO(hlundin): Should `downsamp_temp_len` be corrected for round-off + // errors? I.e., (temp_len + decimation_factor - 1) / decimation_factor? + size_t downsamp_temp_len = temp_len / decimation_factor; + if (downsamp_temp_len > 0) { + WebRtcSpl_DownsampleFast(&input[signal_offset], temp_len, + input_downsampled_, downsamp_temp_len, + filter_coefficients, num_coefficients, + decimation_factor, kCompensateDelay); + } + memset(&input_downsampled_[downsamp_temp_len], 0, + sizeof(int16_t) * (kInputDownsampLength - downsamp_temp_len)); + } else { + WebRtcSpl_DownsampleFast( + &input[signal_offset], input_length - signal_offset, input_downsampled_, + kInputDownsampLength, filter_coefficients, num_coefficients, + decimation_factor, kCompensateDelay); + } +} + +size_t Merge::CorrelateAndPeakSearch(size_t start_position, + size_t input_length, + size_t expand_period) const { + // Calculate correlation without any normalization. + const size_t max_corr_length = kMaxCorrelationLength; + size_t stop_position_downsamp = + std::min(max_corr_length, expand_->max_lag() / (fs_mult_ * 2) + 1); + + int32_t correlation[kMaxCorrelationLength]; + CrossCorrelationWithAutoShift(input_downsampled_, expanded_downsampled_, + kInputDownsampLength, stop_position_downsamp, 1, + correlation); + + // Normalize correlation to 14 bits and copy to a 16-bit array. + const size_t pad_length = expand_->overlap_length() - 1; + const size_t correlation_buffer_size = 2 * pad_length + kMaxCorrelationLength; + std::unique_ptr correlation16( + new int16_t[correlation_buffer_size]); + memset(correlation16.get(), 0, correlation_buffer_size * sizeof(int16_t)); + int16_t* correlation_ptr = &correlation16[pad_length]; + int32_t max_correlation = + WebRtcSpl_MaxAbsValueW32(correlation, stop_position_downsamp); + int norm_shift = std::max(0, 17 - WebRtcSpl_NormW32(max_correlation)); + WebRtcSpl_VectorBitShiftW32ToW16(correlation_ptr, stop_position_downsamp, + correlation, norm_shift); + + // Calculate allowed starting point for peak finding. + // The peak location bestIndex must fulfill two criteria: + // (1) w16_bestIndex + input_length < + // timestamps_per_call_ + expand_->overlap_length(); + // (2) w16_bestIndex + input_length < start_position. + size_t start_index = timestamps_per_call_ + expand_->overlap_length(); + start_index = std::max(start_position, start_index); + start_index = (input_length > start_index) ? 0 : (start_index - input_length); + // Downscale starting index to 4kHz domain. (fs_mult_ * 2 = fs_hz_ / 4000.) + size_t start_index_downsamp = start_index / (fs_mult_ * 2); + + // Calculate a modified `stop_position_downsamp` to account for the increased + // start index `start_index_downsamp` and the effective array length. + size_t modified_stop_pos = + std::min(stop_position_downsamp, + kMaxCorrelationLength + pad_length - start_index_downsamp); + size_t best_correlation_index; + int16_t best_correlation; + static const size_t kNumCorrelationCandidates = 1; + DspHelper::PeakDetection(&correlation_ptr[start_index_downsamp], + modified_stop_pos, kNumCorrelationCandidates, + fs_mult_, &best_correlation_index, + &best_correlation); + // Compensate for modified start index. + best_correlation_index += start_index; + + // Ensure that underrun does not occur for 10ms case => we have to get at + // least 10ms + overlap . (This should never happen thanks to the above + // modification of peak-finding starting point.) + while (((best_correlation_index + input_length) < + (timestamps_per_call_ + expand_->overlap_length())) || + ((best_correlation_index + input_length) < start_position)) { + RTC_DCHECK_NOTREACHED(); // Should never happen. + best_correlation_index += expand_period; // Jump one lag ahead. + } + return best_correlation_index; +} + +size_t Merge::RequiredFutureSamples() { + return fs_hz_ / 100 * num_channels_; // 10 ms. +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/merge.h b/third_party/libwebrtc/modules/audio_coding/neteq/merge.h new file mode 100644 index 0000000000..2f27106bfe --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/merge.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MERGE_H_ +#define MODULES_AUDIO_CODING_NETEQ_MERGE_H_ + +#include "modules/audio_coding/neteq/audio_multi_vector.h" + +namespace webrtc { + +// Forward declarations. +class Expand; +class SyncBuffer; + +// This class handles the transition from expansion to normal operation. +// When a packet is not available for decoding when needed, the expand operation +// is called to generate extrapolation data. If the missing packet arrives, +// i.e., it was just delayed, it can be decoded and appended directly to the +// end of the expanded data (thanks to how the Expand class operates). However, +// if a later packet arrives instead, the loss is a fact, and the new data must +// be stitched together with the end of the expanded data. This stitching is +// what the Merge class does. +class Merge { + public: + Merge(int fs_hz, + size_t num_channels, + Expand* expand, + SyncBuffer* sync_buffer); + virtual ~Merge(); + + Merge(const Merge&) = delete; + Merge& operator=(const Merge&) = delete; + + // The main method to produce the audio data. The decoded data is supplied in + // `input`, having `input_length` samples in total for all channels + // (interleaved). The result is written to `output`. The number of channels + // allocated in `output` defines the number of channels that will be used when + // de-interleaving `input`. + virtual size_t Process(int16_t* input, + size_t input_length, + AudioMultiVector* output); + + virtual size_t RequiredFutureSamples(); + + protected: + const int fs_hz_; + const size_t num_channels_; + + private: + static const int kMaxSampleRate = 48000; + static const size_t kExpandDownsampLength = 100; + static const size_t kInputDownsampLength = 40; + static const size_t kMaxCorrelationLength = 60; + + // Calls `expand_` to get more expansion data to merge with. The data is + // written to `expanded_signal_`. Returns the length of the expanded data, + // while `expand_period` will be the number of samples in one expansion period + // (typically one pitch period). The value of `old_length` will be the number + // of samples that were taken from the `sync_buffer_`. + size_t GetExpandedSignal(size_t* old_length, size_t* expand_period); + + // Analyzes `input` and `expanded_signal` and returns muting factor (Q14) to + // be used on the new data. + int16_t SignalScaling(const int16_t* input, + size_t input_length, + const int16_t* expanded_signal) const; + + // Downsamples `input` (`input_length` samples) and `expanded_signal` to + // 4 kHz sample rate. The downsampled signals are written to + // `input_downsampled_` and `expanded_downsampled_`, respectively. + void Downsample(const int16_t* input, + size_t input_length, + const int16_t* expanded_signal, + size_t expanded_length); + + // Calculates cross-correlation between `input_downsampled_` and + // `expanded_downsampled_`, and finds the correlation maximum. The maximizing + // lag is returned. + size_t CorrelateAndPeakSearch(size_t start_position, + size_t input_length, + size_t expand_period) const; + + const int fs_mult_; // fs_hz_ / 8000. + const size_t timestamps_per_call_; + Expand* expand_; + SyncBuffer* sync_buffer_; + int16_t expanded_downsampled_[kExpandDownsampLength]; + int16_t input_downsampled_[kInputDownsampLength]; + AudioMultiVector expanded_; + std::vector temp_data_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MERGE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/merge_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/merge_unittest.cc new file mode 100644 index 0000000000..d5a55eb056 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/merge_unittest.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for Merge class. + +#include "modules/audio_coding/neteq/merge.h" + +#include +#include + +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +TEST(Merge, CreateAndDestroy) { + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); + Merge merge(fs, channels, &expand, &sync_buffer); +} + +namespace { +// This is the same size that is given to the SyncBuffer object in NetEq. +const size_t kNetEqSyncBufferLengthMs = 720; +} // namespace + +class MergeTest : public testing::TestWithParam { + protected: + MergeTest() + : input_file_(test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000), + test_sample_rate_hz_(8000), + num_channels_(1), + background_noise_(num_channels_), + sync_buffer_(num_channels_, + kNetEqSyncBufferLengthMs * test_sample_rate_hz_ / 1000), + expand_(&background_noise_, + &sync_buffer_, + &random_vector_, + &statistics_, + test_sample_rate_hz_, + num_channels_), + merge_(test_sample_rate_hz_, num_channels_, &expand_, &sync_buffer_) { + input_file_.set_output_rate_hz(test_sample_rate_hz_); + } + + void SetUp() override { + // Fast-forward the input file until there is speech (about 1.1 second into + // the file). + const int speech_start_samples = + static_cast(test_sample_rate_hz_ * 1.1f); + ASSERT_TRUE(input_file_.Seek(speech_start_samples)); + + // Pre-load the sync buffer with speech data. + std::unique_ptr temp(new int16_t[sync_buffer_.Size()]); + ASSERT_TRUE(input_file_.Read(sync_buffer_.Size(), temp.get())); + sync_buffer_.Channel(0).OverwriteAt(temp.get(), sync_buffer_.Size(), 0); + // Move index such that the sync buffer appears to have 5 ms left to play. + sync_buffer_.set_next_index(sync_buffer_.next_index() - + test_sample_rate_hz_ * 5 / 1000); + ASSERT_EQ(1u, num_channels_) << "Fix: Must populate all channels."; + ASSERT_GT(sync_buffer_.FutureLength(), 0u); + } + + test::ResampleInputAudioFile input_file_; + int test_sample_rate_hz_; + size_t num_channels_; + BackgroundNoise background_noise_; + SyncBuffer sync_buffer_; + RandomVector random_vector_; + StatisticsCalculator statistics_; + Expand expand_; + Merge merge_; +}; + +TEST_P(MergeTest, Process) { + AudioMultiVector output(num_channels_); + // Start by calling Expand once, to prime the state. + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + output.Clear(); + // Now call Merge, but with a very short decoded input. Try different length + // if the input. + const size_t input_len = GetParam(); + std::vector input(input_len, 17); + merge_.Process(input.data(), input_len, &output); + EXPECT_GT(output.Size(), 0u); +} + +// Instantiate with values for the input length that are interesting in +// Merge::Downsample. Why are these values interesting? +// - In 8000 Hz sample rate, signal_offset in Merge::Downsample will be 2, so +// the values 1, 2, 3 are just around that value. +// - Also in 8000 Hz, the variable length_limit in the same method will be 80, +// so values 80 and 81 will be on either side of the branch point +// "input_length <= length_limit". +// - Finally, 160 is simply 20 ms in 8000 Hz, which is a common packet size. +INSTANTIATE_TEST_SUITE_P(DifferentInputLengths, + MergeTest, + testing::Values(1, 2, 3, 80, 81, 160)); +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h new file mode 100644 index 0000000000..503f6ac6bd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ + +#include "modules/audio_coding/neteq/buffer_level_filter.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockBufferLevelFilter : public BufferLevelFilter { + public: + MOCK_METHOD(void, + Update, + (size_t buffer_size_samples, int time_stretched_samples)); + MOCK_METHOD(int, filtered_current_level, (), (const)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h new file mode 100644 index 0000000000..2394120e99 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DECODER_DATABASE_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DECODER_DATABASE_H_ + +#include + +#include "modules/audio_coding/neteq/decoder_database.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDecoderDatabase : public DecoderDatabase { + public: + explicit MockDecoderDatabase( + rtc::scoped_refptr factory = nullptr) + : DecoderDatabase(factory, absl::nullopt) {} + ~MockDecoderDatabase() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(bool, Empty, (), (const, override)); + MOCK_METHOD(int, Size, (), (const, override)); + MOCK_METHOD(int, + RegisterPayload, + (int rtp_payload_type, const SdpAudioFormat& audio_format), + (override)); + MOCK_METHOD(int, Remove, (uint8_t rtp_payload_type), (override)); + MOCK_METHOD(void, RemoveAll, (), (override)); + MOCK_METHOD(const DecoderInfo*, + GetDecoderInfo, + (uint8_t rtp_payload_type), + (const, override)); + MOCK_METHOD(int, + SetActiveDecoder, + (uint8_t rtp_payload_type, bool* new_decoder), + (override)); + MOCK_METHOD(AudioDecoder*, GetActiveDecoder, (), (const, override)); + MOCK_METHOD(int, SetActiveCngDecoder, (uint8_t rtp_payload_type), (override)); + MOCK_METHOD(ComfortNoiseDecoder*, GetActiveCngDecoder, (), (const, override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DECODER_DATABASE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_delay_manager.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_delay_manager.h new file mode 100644 index 0000000000..d783f8743b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_delay_manager.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ + +#include "api/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDelayManager : public DelayManager { + public: + MockDelayManager(const MockDelayManager::Config& config, + const TickTimer* tick_timer) + : DelayManager(config, tick_timer) {} + MOCK_METHOD(int, TargetDelayMs, (), (const)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h new file mode 100644 index 0000000000..c60c56d36b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_BUFFER_H_ + +#include "modules/audio_coding/neteq/dtmf_buffer.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDtmfBuffer : public DtmfBuffer { + public: + MockDtmfBuffer(int fs) : DtmfBuffer(fs) {} + ~MockDtmfBuffer() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, Flush, (), (override)); + MOCK_METHOD(int, InsertEvent, (const DtmfEvent& event), (override)); + MOCK_METHOD(bool, + GetEvent, + (uint32_t current_timestamp, DtmfEvent* event), + (override)); + MOCK_METHOD(size_t, Length, (), (const, override)); + MOCK_METHOD(bool, Empty, (), (const, override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_BUFFER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h new file mode 100644 index 0000000000..60de167c29 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_TONE_GENERATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_TONE_GENERATOR_H_ + +#include "modules/audio_coding/neteq/dtmf_tone_generator.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDtmfToneGenerator : public DtmfToneGenerator { + public: + ~MockDtmfToneGenerator() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(int, Init, (int fs, int event, int attenuation), (override)); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(int, + Generate, + (size_t num_samples, AudioMultiVector* output), + (override)); + MOCK_METHOD(bool, initialized, (), (const, override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DTMF_TONE_GENERATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_expand.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_expand.h new file mode 100644 index 0000000000..9d66779021 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_expand.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXPAND_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXPAND_H_ + +#include "modules/audio_coding/neteq/expand.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockExpand : public Expand { + public: + MockExpand(BackgroundNoise* background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels) + : Expand(background_noise, + sync_buffer, + random_vector, + statistics, + fs, + num_channels) {} + ~MockExpand() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(int, Process, (AudioMultiVector * output), (override)); + MOCK_METHOD(void, SetParametersForNormalAfterExpand, (), (override)); + MOCK_METHOD(void, SetParametersForMergeAfterExpand, (), (override)); + MOCK_METHOD(size_t, overlap_length, (), (const, override)); +}; + +} // namespace webrtc + +namespace webrtc { + +class MockExpandFactory : public ExpandFactory { + public: + MOCK_METHOD(Expand*, + Create, + (BackgroundNoise * background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels), + (const, override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_EXPAND_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_histogram.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_histogram.h new file mode 100644 index 0000000000..03abbc1d4b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_histogram.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_HISTOGRAM_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_HISTOGRAM_H_ + +#include "modules/audio_coding/neteq/histogram.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockHistogram : public Histogram { + public: + MockHistogram(size_t num_buckets, int forget_factor) + : Histogram(num_buckets, forget_factor) {} + virtual ~MockHistogram() {} + + MOCK_METHOD(void, Add, (int), (override)); + MOCK_METHOD(int, Quantile, (int), (override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_HISTOGRAM_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_neteq_controller.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_neteq_controller.h new file mode 100644 index 0000000000..6d88e09216 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_neteq_controller.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_ + +#include "api/neteq/neteq_controller.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockNetEqController : public NetEqController { + public: + MockNetEqController() = default; + ~MockNetEqController() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(void, SoftReset, (), (override)); + MOCK_METHOD(NetEq::Operation, + GetDecision, + (const NetEqStatus& neteq_status, bool* reset_decoder), + (override)); + MOCK_METHOD(void, RegisterEmptyPacket, (), (override)); + MOCK_METHOD(void, + SetSampleRate, + (int fs_hz, size_t output_size_samples), + (override)); + MOCK_METHOD(bool, SetMaximumDelay, (int delay_ms), (override)); + MOCK_METHOD(bool, SetMinimumDelay, (int delay_ms), (override)); + MOCK_METHOD(bool, SetBaseMinimumDelay, (int delay_ms), (override)); + MOCK_METHOD(int, GetBaseMinimumDelay, (), (const, override)); + MOCK_METHOD(bool, CngRfc3389On, (), (const, override)); + MOCK_METHOD(bool, CngOff, (), (const, override)); + MOCK_METHOD(void, SetCngOff, (), (override)); + MOCK_METHOD(void, ExpandDecision, (NetEq::Operation operation), (override)); + MOCK_METHOD(void, AddSampleMemory, (int32_t value), (override)); + MOCK_METHOD(int, TargetLevelMs, (), (const, override)); + MOCK_METHOD(absl::optional, + PacketArrived, + (int fs_hz, + bool should_update_stats, + const PacketArrivedInfo& info), + (override)); + MOCK_METHOD(void, NotifyMutedState, (), (override)); + MOCK_METHOD(bool, PeakFound, (), (const, override)); + MOCK_METHOD(int, GetFilteredBufferLevel, (), (const, override)); + MOCK_METHOD(void, set_sample_memory, (int32_t value), (override)); + MOCK_METHOD(size_t, noise_fast_forward, (), (const, override)); + MOCK_METHOD(size_t, packet_length_samples, (), (const, override)); + MOCK_METHOD(void, set_packet_length_samples, (size_t value), (override)); + MOCK_METHOD(void, set_prev_time_scale, (bool value), (override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h new file mode 100644 index 0000000000..48357ea466 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PACKET_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PACKET_BUFFER_H_ + +#include "modules/audio_coding/neteq/packet_buffer.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockPacketBuffer : public PacketBuffer { + public: + MockPacketBuffer(size_t max_number_of_packets, const TickTimer* tick_timer) + : PacketBuffer(max_number_of_packets, tick_timer) {} + ~MockPacketBuffer() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, Flush, (StatisticsCalculator * stats), (override)); + MOCK_METHOD(void, + PartialFlush, + (int target_level_ms, + size_t sample_rate, + size_t last_decoded_length, + StatisticsCalculator* stats), + (override)); + MOCK_METHOD(bool, Empty, (), (const, override)); + MOCK_METHOD(int, + InsertPacket, + (Packet && packet, + StatisticsCalculator* stats, + size_t last_decoded_length, + size_t sample_rate, + int target_level_ms, + const DecoderDatabase& decoder_database), + (override)); + MOCK_METHOD(int, + InsertPacketList, + (PacketList * packet_list, + const DecoderDatabase& decoder_database, + absl::optional* current_rtp_payload_type, + absl::optional* current_cng_rtp_payload_type, + StatisticsCalculator* stats, + size_t last_decoded_length, + size_t sample_rate, + int target_level_ms), + (override)); + MOCK_METHOD(int, + NextTimestamp, + (uint32_t * next_timestamp), + (const, override)); + MOCK_METHOD(int, + NextHigherTimestamp, + (uint32_t timestamp, uint32_t* next_timestamp), + (const, override)); + MOCK_METHOD(const Packet*, PeekNextPacket, (), (const, override)); + MOCK_METHOD(absl::optional, GetNextPacket, (), (override)); + MOCK_METHOD(int, + DiscardNextPacket, + (StatisticsCalculator * stats), + (override)); + MOCK_METHOD(void, + DiscardOldPackets, + (uint32_t timestamp_limit, + uint32_t horizon_samples, + StatisticsCalculator* stats), + (override)); + MOCK_METHOD(void, + DiscardAllOldPackets, + (uint32_t timestamp_limit, StatisticsCalculator* stats), + (override)); + MOCK_METHOD(size_t, NumPacketsInBuffer, (), (const, override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_PACKET_BUFFER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h new file mode 100644 index 0000000000..9daf571a80 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ + +#include "modules/audio_coding/neteq/red_payload_splitter.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockRedPayloadSplitter : public RedPayloadSplitter { + public: + MOCK_METHOD(bool, SplitRed, (PacketList * packet_list), (override)); + MOCK_METHOD(void, + CheckRedPayloads, + (PacketList * packet_list, + const DecoderDatabase& decoder_database), + (override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_RED_PAYLOAD_SPLITTER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h new file mode 100644 index 0000000000..f8812478d6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_STATISTICS_CALCULATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_STATISTICS_CALCULATOR_H_ + +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockStatisticsCalculator : public StatisticsCalculator { + public: + MOCK_METHOD(void, PacketsDiscarded, (size_t num_packets), (override)); + MOCK_METHOD(void, + SecondaryPacketsDiscarded, + (size_t num_packets), + (override)); + MOCK_METHOD(void, RelativePacketArrivalDelay, (size_t delay_ms), (override)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_STATISTICS_CALCULATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.cc b/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.cc new file mode 100644 index 0000000000..04cc5b52e8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.cc @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/nack_tracker.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/experiments/struct_parameters_parser.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace { + +const int kDefaultSampleRateKhz = 48; +const int kMaxPacketSizeMs = 120; +constexpr char kNackTrackerConfigFieldTrial[] = + "WebRTC-Audio-NetEqNackTrackerConfig"; + +} // namespace + +NackTracker::Config::Config() { + auto parser = StructParametersParser::Create( + "packet_loss_forget_factor", &packet_loss_forget_factor, + "ms_per_loss_percent", &ms_per_loss_percent, "never_nack_multiple_times", + &never_nack_multiple_times, "require_valid_rtt", &require_valid_rtt, + "max_loss_rate", &max_loss_rate); + parser->Parse( + webrtc::field_trial::FindFullName(kNackTrackerConfigFieldTrial)); + RTC_LOG(LS_INFO) << "Nack tracker config:" + " packet_loss_forget_factor=" + << packet_loss_forget_factor + << " ms_per_loss_percent=" << ms_per_loss_percent + << " never_nack_multiple_times=" << never_nack_multiple_times + << " require_valid_rtt=" << require_valid_rtt + << " max_loss_rate=" << max_loss_rate; +} + +NackTracker::NackTracker() + : sequence_num_last_received_rtp_(0), + timestamp_last_received_rtp_(0), + any_rtp_received_(false), + sequence_num_last_decoded_rtp_(0), + timestamp_last_decoded_rtp_(0), + any_rtp_decoded_(false), + sample_rate_khz_(kDefaultSampleRateKhz), + max_nack_list_size_(kNackListSizeLimit) {} + +NackTracker::~NackTracker() = default; + +void NackTracker::UpdateSampleRate(int sample_rate_hz) { + RTC_DCHECK_GT(sample_rate_hz, 0); + sample_rate_khz_ = sample_rate_hz / 1000; +} + +void NackTracker::UpdateLastReceivedPacket(uint16_t sequence_number, + uint32_t timestamp) { + // Just record the value of sequence number and timestamp if this is the + // first packet. + if (!any_rtp_received_) { + sequence_num_last_received_rtp_ = sequence_number; + timestamp_last_received_rtp_ = timestamp; + any_rtp_received_ = true; + // If no packet is decoded, to have a reasonable estimate of time-to-play + // use the given values. + if (!any_rtp_decoded_) { + sequence_num_last_decoded_rtp_ = sequence_number; + timestamp_last_decoded_rtp_ = timestamp; + } + return; + } + + if (sequence_number == sequence_num_last_received_rtp_) + return; + + // Received RTP should not be in the list. + nack_list_.erase(sequence_number); + + // If this is an old sequence number, no more action is required, return. + if (IsNewerSequenceNumber(sequence_num_last_received_rtp_, sequence_number)) + return; + + UpdatePacketLossRate(sequence_number - sequence_num_last_received_rtp_ - 1); + + UpdateList(sequence_number, timestamp); + + sequence_num_last_received_rtp_ = sequence_number; + timestamp_last_received_rtp_ = timestamp; + LimitNackListSize(); +} + +absl::optional NackTracker::GetSamplesPerPacket( + uint16_t sequence_number_current_received_rtp, + uint32_t timestamp_current_received_rtp) const { + uint32_t timestamp_increase = + timestamp_current_received_rtp - timestamp_last_received_rtp_; + uint16_t sequence_num_increase = + sequence_number_current_received_rtp - sequence_num_last_received_rtp_; + + int samples_per_packet = timestamp_increase / sequence_num_increase; + if (samples_per_packet == 0 || + samples_per_packet > kMaxPacketSizeMs * sample_rate_khz_) { + // Not a valid samples per packet. + return absl::nullopt; + } + return samples_per_packet; +} + +void NackTracker::UpdateList(uint16_t sequence_number_current_received_rtp, + uint32_t timestamp_current_received_rtp) { + if (!IsNewerSequenceNumber(sequence_number_current_received_rtp, + sequence_num_last_received_rtp_ + 1)) { + return; + } + RTC_DCHECK(!any_rtp_decoded_ || + IsNewerSequenceNumber(sequence_number_current_received_rtp, + sequence_num_last_decoded_rtp_)); + + absl::optional samples_per_packet = GetSamplesPerPacket( + sequence_number_current_received_rtp, timestamp_current_received_rtp); + if (!samples_per_packet) { + return; + } + + for (uint16_t n = sequence_num_last_received_rtp_ + 1; + IsNewerSequenceNumber(sequence_number_current_received_rtp, n); ++n) { + uint32_t timestamp = EstimateTimestamp(n, *samples_per_packet); + NackElement nack_element(TimeToPlay(timestamp), timestamp); + nack_list_.insert(nack_list_.end(), std::make_pair(n, nack_element)); + } +} + +uint32_t NackTracker::EstimateTimestamp(uint16_t sequence_num, + int samples_per_packet) { + uint16_t sequence_num_diff = sequence_num - sequence_num_last_received_rtp_; + return sequence_num_diff * samples_per_packet + timestamp_last_received_rtp_; +} + +void NackTracker::UpdateEstimatedPlayoutTimeBy10ms() { + while (!nack_list_.empty() && + nack_list_.begin()->second.time_to_play_ms <= 10) + nack_list_.erase(nack_list_.begin()); + + for (NackList::iterator it = nack_list_.begin(); it != nack_list_.end(); ++it) + it->second.time_to_play_ms -= 10; +} + +void NackTracker::UpdateLastDecodedPacket(uint16_t sequence_number, + uint32_t timestamp) { + if (IsNewerSequenceNumber(sequence_number, sequence_num_last_decoded_rtp_) || + !any_rtp_decoded_) { + sequence_num_last_decoded_rtp_ = sequence_number; + timestamp_last_decoded_rtp_ = timestamp; + // Packets in the list with sequence numbers less than the + // sequence number of the decoded RTP should be removed from the lists. + // They will be discarded by the jitter buffer if they arrive. + nack_list_.erase(nack_list_.begin(), + nack_list_.upper_bound(sequence_num_last_decoded_rtp_)); + + // Update estimated time-to-play. + for (NackList::iterator it = nack_list_.begin(); it != nack_list_.end(); + ++it) + it->second.time_to_play_ms = TimeToPlay(it->second.estimated_timestamp); + } else { + RTC_DCHECK_EQ(sequence_number, sequence_num_last_decoded_rtp_); + + // Same sequence number as before. 10 ms is elapsed, update estimations for + // time-to-play. + UpdateEstimatedPlayoutTimeBy10ms(); + + // Update timestamp for better estimate of time-to-play, for packets which + // are added to NACK list later on. + timestamp_last_decoded_rtp_ += sample_rate_khz_ * 10; + } + any_rtp_decoded_ = true; +} + +NackTracker::NackList NackTracker::GetNackList() const { + return nack_list_; +} + +void NackTracker::Reset() { + nack_list_.clear(); + + sequence_num_last_received_rtp_ = 0; + timestamp_last_received_rtp_ = 0; + any_rtp_received_ = false; + sequence_num_last_decoded_rtp_ = 0; + timestamp_last_decoded_rtp_ = 0; + any_rtp_decoded_ = false; + sample_rate_khz_ = kDefaultSampleRateKhz; +} + +void NackTracker::SetMaxNackListSize(size_t max_nack_list_size) { + RTC_CHECK_GT(max_nack_list_size, 0); + // Ugly hack to get around the problem of passing static consts by reference. + const size_t kNackListSizeLimitLocal = NackTracker::kNackListSizeLimit; + RTC_CHECK_LE(max_nack_list_size, kNackListSizeLimitLocal); + + max_nack_list_size_ = max_nack_list_size; + LimitNackListSize(); +} + +void NackTracker::LimitNackListSize() { + uint16_t limit = sequence_num_last_received_rtp_ - + static_cast(max_nack_list_size_) - 1; + nack_list_.erase(nack_list_.begin(), nack_list_.upper_bound(limit)); +} + +int64_t NackTracker::TimeToPlay(uint32_t timestamp) const { + uint32_t timestamp_increase = timestamp - timestamp_last_decoded_rtp_; + return timestamp_increase / sample_rate_khz_; +} + +// We don't erase elements with time-to-play shorter than round-trip-time. +std::vector NackTracker::GetNackList(int64_t round_trip_time_ms) { + RTC_DCHECK_GE(round_trip_time_ms, 0); + std::vector sequence_numbers; + if (round_trip_time_ms == 0) { + if (config_.require_valid_rtt) { + return sequence_numbers; + } else { + round_trip_time_ms = config_.default_rtt_ms; + } + } + if (packet_loss_rate_ > + static_cast(config_.max_loss_rate * (1 << 30))) { + return sequence_numbers; + } + // The estimated packet loss is between 0 and 1, so we need to multiply by 100 + // here. + int max_wait_ms = + 100.0 * config_.ms_per_loss_percent * packet_loss_rate_ / (1 << 30); + for (NackList::const_iterator it = nack_list_.begin(); it != nack_list_.end(); + ++it) { + int64_t time_since_packet_ms = + (timestamp_last_received_rtp_ - it->second.estimated_timestamp) / + sample_rate_khz_; + if (it->second.time_to_play_ms > round_trip_time_ms || + time_since_packet_ms + round_trip_time_ms < max_wait_ms) + sequence_numbers.push_back(it->first); + } + if (config_.never_nack_multiple_times) { + nack_list_.clear(); + } + return sequence_numbers; +} + +void NackTracker::UpdatePacketLossRate(int packets_lost) { + const uint64_t alpha_q30 = (1 << 30) * config_.packet_loss_forget_factor; + // Exponential filter. + packet_loss_rate_ = (alpha_q30 * packet_loss_rate_) >> 30; + for (int i = 0; i < packets_lost; ++i) { + packet_loss_rate_ = + ((alpha_q30 * packet_loss_rate_) >> 30) + ((1 << 30) - alpha_q30); + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.h b/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.h new file mode 100644 index 0000000000..14ba2166d1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.h @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_NACK_TRACKER_H_ +#define MODULES_AUDIO_CODING_NETEQ_NACK_TRACKER_H_ + +#include +#include + +#include +#include + +#include "absl/types/optional.h" +#include "modules/include/module_common_types_public.h" +#include "rtc_base/gtest_prod_util.h" + +// +// The NackTracker class keeps track of the lost packets, an estimate of +// time-to-play for each packet is also given. +// +// Every time a packet is pushed into NetEq, LastReceivedPacket() has to be +// called to update the NACK list. +// +// Every time 10ms audio is pulled from NetEq LastDecodedPacket() should be +// called, and time-to-play is updated at that moment. +// +// If packet N is received, any packet prior to N which has not arrived is +// considered lost, and should be labeled as "missing" (the size of +// the list might be limited and older packet eliminated from the list). +// +// The NackTracker class has to know about the sample rate of the packets to +// compute time-to-play. So sample rate should be set as soon as the first +// packet is received. If there is a change in the receive codec (sender changes +// codec) then NackTracker should be reset. This is because NetEQ would flush +// its buffer and re-transmission is meaning less for old packet. Therefore, in +// that case, after reset the sampling rate has to be updated. +// +// Thread Safety +// ============= +// Please note that this class in not thread safe. The class must be protected +// if different APIs are called from different threads. +// +namespace webrtc { + +class NackTracker { + public: + // A limit for the size of the NACK list. + static const size_t kNackListSizeLimit = 500; // 10 seconds for 20 ms frame + // packets. + NackTracker(); + ~NackTracker(); + + // Set a maximum for the size of the NACK list. If the last received packet + // has sequence number of N, then NACK list will not contain any element + // with sequence number earlier than N - `max_nack_list_size`. + // + // The largest maximum size is defined by `kNackListSizeLimit` + void SetMaxNackListSize(size_t max_nack_list_size); + + // Set the sampling rate. + // + // If associated sampling rate of the received packets is changed, call this + // function to update sampling rate. Note that if there is any change in + // received codec then NetEq will flush its buffer and NACK has to be reset. + // After Reset() is called sampling rate has to be set. + void UpdateSampleRate(int sample_rate_hz); + + // Update the sequence number and the timestamp of the last decoded RTP. This + // API should be called every time 10 ms audio is pulled from NetEq. + void UpdateLastDecodedPacket(uint16_t sequence_number, uint32_t timestamp); + + // Update the sequence number and the timestamp of the last received RTP. This + // API should be called every time a packet pushed into ACM. + void UpdateLastReceivedPacket(uint16_t sequence_number, uint32_t timestamp); + + // Get a list of "missing" packets which have expected time-to-play larger + // than the given round-trip-time (in milliseconds). + // Note: Late packets are not included. + // Calling this method multiple times may give different results, since the + // internal nack list may get flushed if never_nack_multiple_times_ is true. + std::vector GetNackList(int64_t round_trip_time_ms); + + // Reset to default values. The NACK list is cleared. + // `max_nack_list_size_` preserves its value. + void Reset(); + + // Returns the estimated packet loss rate in Q30, for testing only. + uint32_t GetPacketLossRateForTest() { return packet_loss_rate_; } + + private: + // This test need to access the private method GetNackList(). + FRIEND_TEST_ALL_PREFIXES(NackTrackerTest, EstimateTimestampAndTimeToPlay); + + // Options that can be configured via field trial. + struct Config { + Config(); + + // The exponential decay factor used to estimate the packet loss rate. + double packet_loss_forget_factor = 0.996; + // How many additional ms we are willing to wait (at most) for nacked + // packets for each additional percentage of packet loss. + int ms_per_loss_percent = 20; + // If true, never nack packets more than once. + bool never_nack_multiple_times = false; + // Only nack if the RTT is valid. + bool require_valid_rtt = false; + // Default RTT to use unless `require_valid_rtt` is set. + int default_rtt_ms = 100; + // Do not nack if the loss rate is above this value. + double max_loss_rate = 1.0; + }; + + struct NackElement { + NackElement(int64_t initial_time_to_play_ms, uint32_t initial_timestamp) + : time_to_play_ms(initial_time_to_play_ms), + estimated_timestamp(initial_timestamp) {} + + // Estimated time (ms) left for this packet to be decoded. This estimate is + // updated every time jitter buffer decodes a packet. + int64_t time_to_play_ms; + + // A guess about the timestamp of the missing packet, it is used for + // estimation of `time_to_play_ms`. The estimate might be slightly wrong if + // there has been frame-size change since the last received packet and the + // missing packet. However, the risk of this is low, and in case of such + // errors, there will be a minor misestimation in time-to-play of missing + // packets. This will have a very minor effect on NACK performance. + uint32_t estimated_timestamp; + }; + + class NackListCompare { + public: + bool operator()(uint16_t sequence_number_old, + uint16_t sequence_number_new) const { + return IsNewerSequenceNumber(sequence_number_new, sequence_number_old); + } + }; + + typedef std::map NackList; + + // This API is used only for testing to assess whether time-to-play is + // computed correctly. + NackList GetNackList() const; + + // This function subtracts 10 ms of time-to-play for all packets in NACK list. + // This is called when 10 ms elapsed with no new RTP packet decoded. + void UpdateEstimatedPlayoutTimeBy10ms(); + + // Returns a valid number of samples per packet given the current received + // sequence number and timestamp or nullopt of none could be computed. + absl::optional GetSamplesPerPacket( + uint16_t sequence_number_current_received_rtp, + uint32_t timestamp_current_received_rtp) const; + + // Given the `sequence_number_current_received_rtp` of currently received RTP + // update the list. Packets that are older than the received packet are added + // to the nack list. + void UpdateList(uint16_t sequence_number_current_received_rtp, + uint32_t timestamp_current_received_rtp); + + // Packets which have sequence number older that + // `sequence_num_last_received_rtp_` - `max_nack_list_size_` are removed + // from the NACK list. + void LimitNackListSize(); + + // Estimate timestamp of a missing packet given its sequence number. + uint32_t EstimateTimestamp(uint16_t sequence_number, int samples_per_packet); + + // Compute time-to-play given a timestamp. + int64_t TimeToPlay(uint32_t timestamp) const; + + // Updates the estimated packet lost rate. + void UpdatePacketLossRate(int packets_lost); + + const Config config_; + + // Valid if a packet is received. + uint16_t sequence_num_last_received_rtp_; + uint32_t timestamp_last_received_rtp_; + bool any_rtp_received_; // If any packet received. + + // Valid if a packet is decoded. + uint16_t sequence_num_last_decoded_rtp_; + uint32_t timestamp_last_decoded_rtp_; + bool any_rtp_decoded_; // If any packet decoded. + + int sample_rate_khz_; // Sample rate in kHz. + + // A list of missing packets to be retransmitted. Components of the list + // contain the sequence number of missing packets and the estimated time that + // each pack is going to be played out. + NackList nack_list_; + + // NACK list will not keep track of missing packets prior to + // `sequence_num_last_received_rtp_` - `max_nack_list_size_`. + size_t max_nack_list_size_; + + // Current estimate of the packet loss rate in Q30. + uint32_t packet_loss_rate_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_NACK_TRACKER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker_unittest.cc new file mode 100644 index 0000000000..bcc5120ff3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker_unittest.cc @@ -0,0 +1,565 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/nack_tracker.h" + +#include + +#include +#include + +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const int kSampleRateHz = 16000; +const int kPacketSizeMs = 30; +const uint32_t kTimestampIncrement = 480; // 30 ms. +const int64_t kShortRoundTripTimeMs = 1; + +bool IsNackListCorrect(const std::vector& nack_list, + const uint16_t* lost_sequence_numbers, + size_t num_lost_packets) { + if (nack_list.size() != num_lost_packets) + return false; + + if (num_lost_packets == 0) + return true; + + for (size_t k = 0; k < nack_list.size(); ++k) { + int seq_num = nack_list[k]; + bool seq_num_matched = false; + for (size_t n = 0; n < num_lost_packets; ++n) { + if (seq_num == lost_sequence_numbers[n]) { + seq_num_matched = true; + break; + } + } + if (!seq_num_matched) + return false; + } + return true; +} + +} // namespace + +TEST(NackTrackerTest, EmptyListWhenNoPacketLoss) { + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + + int seq_num = 1; + uint32_t timestamp = 0; + + std::vector nack_list; + for (int n = 0; n < 100; n++) { + nack.UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + seq_num++; + timestamp += kTimestampIncrement; + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } +} + +TEST(NackTrackerTest, LatePacketsMovedToNackThenNackListDoesNotChange) { + const uint16_t kSequenceNumberLostPackets[] = {2, 3, 4, 5, 6, 7, 8, 9}; + static const int kNumAllLostPackets = sizeof(kSequenceNumberLostPackets) / + sizeof(kSequenceNumberLostPackets[0]); + + for (int k = 0; k < 2; k++) { // Two iteration with/without wrap around. + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + + uint16_t sequence_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; n++) { + sequence_num_lost_packets[n] = + kSequenceNumberLostPackets[n] + + k * 65531; // Have wrap around in sequence numbers for |k == 1|. + } + uint16_t seq_num = sequence_num_lost_packets[0] - 1; + + uint32_t timestamp = 0; + std::vector nack_list; + + nack.UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + seq_num = sequence_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp += kTimestampIncrement * (kNumAllLostPackets + 1); + int num_lost_packets = std::max(0, kNumAllLostPackets); + + nack.UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect(nack_list, sequence_num_lost_packets, + num_lost_packets)); + seq_num++; + timestamp += kTimestampIncrement; + num_lost_packets++; + + for (int n = 0; n < 100; ++n) { + nack.UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect(nack_list, sequence_num_lost_packets, + kNumAllLostPackets)); + seq_num++; + timestamp += kTimestampIncrement; + } + } +} + +TEST(NackTrackerTest, ArrivedPacketsAreRemovedFromNackList) { + const uint16_t kSequenceNumberLostPackets[] = {2, 3, 4, 5, 6, 7, 8, 9}; + static const int kNumAllLostPackets = sizeof(kSequenceNumberLostPackets) / + sizeof(kSequenceNumberLostPackets[0]); + + for (int k = 0; k < 2; ++k) { // Two iteration with/without wrap around. + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + + uint16_t sequence_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; ++n) { + sequence_num_lost_packets[n] = kSequenceNumberLostPackets[n] + + k * 65531; // Wrap around for |k == 1|. + } + + uint16_t seq_num = sequence_num_lost_packets[0] - 1; + uint32_t timestamp = 0; + + nack.UpdateLastReceivedPacket(seq_num, timestamp); + std::vector nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + size_t index_retransmitted_rtp = 0; + uint32_t timestamp_retransmitted_rtp = timestamp + kTimestampIncrement; + + seq_num = sequence_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp += kTimestampIncrement * (kNumAllLostPackets + 1); + size_t num_lost_packets = kNumAllLostPackets; + for (int n = 0; n < kNumAllLostPackets; ++n) { + // Number of lost packets does not change for the first + // |kNackThreshold + 1| packets, one is added to the list and one is + // removed. Thereafter, the list shrinks every iteration. + if (n >= 1) + num_lost_packets--; + + nack.UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &sequence_num_lost_packets[index_retransmitted_rtp], + num_lost_packets)); + seq_num++; + timestamp += kTimestampIncrement; + + // Retransmission of a lost RTP. + nack.UpdateLastReceivedPacket( + sequence_num_lost_packets[index_retransmitted_rtp], + timestamp_retransmitted_rtp); + index_retransmitted_rtp++; + timestamp_retransmitted_rtp += kTimestampIncrement; + + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &sequence_num_lost_packets[index_retransmitted_rtp], + num_lost_packets - 1)); // One less lost packet in the list. + } + ASSERT_TRUE(nack_list.empty()); + } +} + +// Assess if estimation of timestamps and time-to-play is correct. Introduce all +// combinations that timestamps and sequence numbers might have wrap around. +TEST(NackTrackerTest, EstimateTimestampAndTimeToPlay) { + const uint16_t kLostPackets[] = {2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15}; + static const int kNumAllLostPackets = + sizeof(kLostPackets) / sizeof(kLostPackets[0]); + + for (int k = 0; k < 4; ++k) { + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + + // Sequence number wrap around if `k` is 2 or 3; + int seq_num_offset = (k < 2) ? 0 : 65531; + + // Timestamp wrap around if `k` is 1 or 3. + uint32_t timestamp_offset = + (k & 0x1) ? static_cast(0xffffffff) - 6 : 0; + + uint32_t timestamp_lost_packets[kNumAllLostPackets]; + uint16_t seq_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; ++n) { + timestamp_lost_packets[n] = + timestamp_offset + kLostPackets[n] * kTimestampIncrement; + seq_num_lost_packets[n] = seq_num_offset + kLostPackets[n]; + } + + // We and to push two packets before lost burst starts. + uint16_t seq_num = seq_num_lost_packets[0] - 2; + uint32_t timestamp = timestamp_lost_packets[0] - 2 * kTimestampIncrement; + + const uint16_t first_seq_num = seq_num; + const uint32_t first_timestamp = timestamp; + + // Two consecutive packets to have a correct estimate of timestamp increase. + nack.UpdateLastReceivedPacket(seq_num, timestamp); + seq_num++; + timestamp += kTimestampIncrement; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + // A packet after the last one which is supposed to be lost. + seq_num = seq_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp = + timestamp_lost_packets[kNumAllLostPackets - 1] + kTimestampIncrement; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + NackTracker::NackList nack_list = nack.GetNackList(); + EXPECT_EQ(static_cast(kNumAllLostPackets), nack_list.size()); + + // Pretend the first packet is decoded. + nack.UpdateLastDecodedPacket(first_seq_num, first_timestamp); + nack_list = nack.GetNackList(); + + NackTracker::NackList::iterator it = nack_list.begin(); + while (it != nack_list.end()) { + seq_num = it->first - seq_num_offset; + int index = seq_num - kLostPackets[0]; + EXPECT_EQ(timestamp_lost_packets[index], it->second.estimated_timestamp); + EXPECT_EQ((index + 2) * kPacketSizeMs, it->second.time_to_play_ms); + ++it; + } + + // Pretend 10 ms is passed, and we had pulled audio from NetEq, it still + // reports the same sequence number as decoded, time-to-play should be + // updated by 10 ms. + nack.UpdateLastDecodedPacket(first_seq_num, first_timestamp); + nack_list = nack.GetNackList(); + it = nack_list.begin(); + while (it != nack_list.end()) { + seq_num = it->first - seq_num_offset; + int index = seq_num - kLostPackets[0]; + EXPECT_EQ((index + 2) * kPacketSizeMs - 10, it->second.time_to_play_ms); + ++it; + } + } +} + +TEST(NackTrackerTest, + MissingPacketsPriorToLastDecodedRtpShouldNotBeInNackList) { + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65531; // Wrap around if `m` is 1. + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + + // Two consecutive packets to have a correct estimate of timestamp increase. + uint16_t seq_num = 0; + nack.UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + seq_num++; + nack.UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + + // Skip 10 packets (larger than NACK threshold). + const int kNumLostPackets = 10; + seq_num += kNumLostPackets + 1; + nack.UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + + const size_t kExpectedListSize = kNumLostPackets; + std::vector nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + + for (int k = 0; k < 2; ++k) { + // Decoding of the first and the second arrived packets. + for (int n = 0; n < kPacketSizeMs / 10; ++n) { + nack.UpdateLastDecodedPacket(seq_num_offset + k, + k * kTimestampIncrement); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + } + } + + // Decoding of the last received packet. + nack.UpdateLastDecodedPacket(seq_num + seq_num_offset, + seq_num * kTimestampIncrement); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + // Make sure list of late packets is also empty. To check that, push few + // packets, if the late list is not empty its content will pop up in NACK + // list. + for (int n = 0; n < 10; ++n) { + seq_num++; + nack.UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } + } +} + +TEST(NackTrackerTest, Reset) { + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + + // Two consecutive packets to have a correct estimate of timestamp increase. + uint16_t seq_num = 0; + nack.UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + seq_num++; + nack.UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + + // Skip 10 packets (larger than NACK threshold). + const int kNumLostPackets = 10; + seq_num += kNumLostPackets + 1; + nack.UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + + const size_t kExpectedListSize = kNumLostPackets; + std::vector nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + + nack.Reset(); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); +} + +TEST(NackTrackerTest, ListSizeAppliedFromBeginning) { + const size_t kNackListSize = 10; + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65525; // Wrap around if `m` is 1. + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + nack.SetMaxNackListSize(kNackListSize); + + uint16_t seq_num = seq_num_offset; + uint32_t timestamp = 0x12345678; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t num_lost_packets = kNackListSize + 5; + + seq_num += num_lost_packets + 1; + timestamp += (num_lost_packets + 1) * kTimestampIncrement; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + std::vector nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kNackListSize, nack_list.size()); + } +} + +TEST(NackTrackerTest, ChangeOfListSizeAppliedAndOldElementsRemoved) { + const size_t kNackListSize = 10; + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65525; // Wrap around if `m` is 1. + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + + uint16_t seq_num = seq_num_offset; + uint32_t timestamp = 0x87654321; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t num_lost_packets = kNackListSize + 5; + + std::unique_ptr seq_num_lost(new uint16_t[num_lost_packets]); + for (int n = 0; n < num_lost_packets; ++n) { + seq_num_lost[n] = ++seq_num; + } + + ++seq_num; + timestamp += (num_lost_packets + 1) * kTimestampIncrement; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + size_t expected_size = num_lost_packets; + + std::vector nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(expected_size, nack_list.size()); + + nack.SetMaxNackListSize(kNackListSize); + expected_size = kNackListSize; + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &seq_num_lost[num_lost_packets - kNackListSize], + expected_size)); + + // NACK list should shrink. + for (size_t n = 1; n < kNackListSize; ++n) { + ++seq_num; + timestamp += kTimestampIncrement; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + --expected_size; + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &seq_num_lost[num_lost_packets - kNackListSize + n], + expected_size)); + } + + // After this packet, NACK list should be empty. + ++seq_num; + timestamp += kTimestampIncrement; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack.GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } +} + +TEST(NackTrackerTest, RoudTripTimeIsApplied) { + const int kNackListSize = 200; + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + nack.SetMaxNackListSize(kNackListSize); + + uint16_t seq_num = 0; + uint32_t timestamp = 0x87654321; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t kNumLostPackets = 5; + + seq_num += (1 + kNumLostPackets); + timestamp += (1 + kNumLostPackets) * kTimestampIncrement; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + // Expected time-to-play are: + // kPacketSizeMs - 10, 2*kPacketSizeMs - 10, 3*kPacketSizeMs - 10, ... + // + // sequence number: 1, 2, 3, 4, 5 + // time-to-play: 20, 50, 80, 110, 140 + // + std::vector nack_list = nack.GetNackList(100); + ASSERT_EQ(2u, nack_list.size()); + EXPECT_EQ(4, nack_list[0]); + EXPECT_EQ(5, nack_list[1]); +} + +// Set never_nack_multiple_times to true with a field trial and verify that +// packets are not nacked multiple times. +TEST(NackTrackerTest, DoNotNackMultipleTimes) { + test::ScopedFieldTrials field_trials( + "WebRTC-Audio-NetEqNackTrackerConfig/" + "packet_loss_forget_factor:0.996,ms_per_loss_percent:20," + "never_nack_multiple_times:true/"); + const int kNackListSize = 200; + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + nack.SetMaxNackListSize(kNackListSize); + + uint16_t seq_num = 0; + uint32_t timestamp = 0x87654321; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + uint16_t kNumLostPackets = 3; + + seq_num += (1 + kNumLostPackets); + timestamp += (1 + kNumLostPackets) * kTimestampIncrement; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + + std::vector nack_list = nack.GetNackList(10); + ASSERT_EQ(3u, nack_list.size()); + EXPECT_EQ(1, nack_list[0]); + EXPECT_EQ(2, nack_list[1]); + EXPECT_EQ(3, nack_list[2]); + // When we get the nack list again, it should be empty. + std::vector nack_list2 = nack.GetNackList(10); + EXPECT_TRUE(nack_list2.empty()); +} + +// Test if estimated packet loss rate is correct. +TEST(NackTrackerTest, PacketLossRateCorrect) { + const int kNackListSize = 200; + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + nack.SetMaxNackListSize(kNackListSize); + uint16_t seq_num = 0; + uint32_t timestamp = 0x87654321; + auto add_packet = [&nack, &seq_num, ×tamp](bool received) { + if (received) { + nack.UpdateLastReceivedPacket(seq_num, timestamp); + } + seq_num++; + timestamp += kTimestampIncrement; + }; + // Add some packets, but every fourth packet is lost. + for (int i = 0; i < 300; i++) { + add_packet(true); + add_packet(true); + add_packet(true); + add_packet(false); + } + // 1 << 28 is 0.25 in Q30. We expect the packet loss estimate to be within + // 0.01 of that. + EXPECT_NEAR(nack.GetPacketLossRateForTest(), 1 << 28, (1 << 30) / 100); +} + +TEST(NackTrackerTest, DoNotNackAfterDtx) { + const int kNackListSize = 200; + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + nack.SetMaxNackListSize(kNackListSize); + uint16_t seq_num = 0; + uint32_t timestamp = 0x87654321; + nack.UpdateLastReceivedPacket(seq_num, timestamp); + EXPECT_TRUE(nack.GetNackList(0).empty()); + constexpr int kDtxPeriod = 400; + nack.UpdateLastReceivedPacket(seq_num + 2, + timestamp + kDtxPeriod * kSampleRateHz / 1000); + EXPECT_TRUE(nack.GetNackList(0).empty()); +} + +TEST(NackTrackerTest, DoNotNackIfLossRateIsTooHigh) { + test::ScopedFieldTrials field_trials( + "WebRTC-Audio-NetEqNackTrackerConfig/max_loss_rate:0.4/"); + const int kNackListSize = 200; + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + nack.SetMaxNackListSize(kNackListSize); + uint16_t seq_num = 0; + uint32_t timestamp = 0x87654321; + auto add_packet = [&nack, &seq_num, ×tamp](bool received) { + if (received) { + nack.UpdateLastReceivedPacket(seq_num, timestamp); + } + seq_num++; + timestamp += kTimestampIncrement; + }; + for (int i = 0; i < 500; i++) { + add_packet(true); + add_packet(false); + } + // Expect 50% loss rate which is higher that the configured maximum 40%. + EXPECT_NEAR(nack.GetPacketLossRateForTest(), 1 << 29, (1 << 30) / 100); + EXPECT_TRUE(nack.GetNackList(0).empty()); +} + +TEST(NackTrackerTest, OnlyNackIfRttIsValid) { + test::ScopedFieldTrials field_trials( + "WebRTC-Audio-NetEqNackTrackerConfig/require_valid_rtt:true/"); + const int kNackListSize = 200; + NackTracker nack; + nack.UpdateSampleRate(kSampleRateHz); + nack.SetMaxNackListSize(kNackListSize); + uint16_t seq_num = 0; + uint32_t timestamp = 0x87654321; + auto add_packet = [&nack, &seq_num, ×tamp](bool received) { + if (received) { + nack.UpdateLastReceivedPacket(seq_num, timestamp); + } + seq_num++; + timestamp += kTimestampIncrement; + }; + add_packet(true); + add_packet(false); + add_packet(true); + EXPECT_TRUE(nack.GetNackList(0).empty()); + EXPECT_FALSE(nack.GetNackList(10).empty()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc new file mode 100644 index 0000000000..cf310d1efb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc @@ -0,0 +1,313 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Test to verify correct operation when using the decoder-internal PLC. + +#include +#include +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h" +#include "modules/audio_coding/neteq/tools/audio_checksum.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "modules/audio_coding/neteq/tools/encode_neteq_input.h" +#include "modules/audio_coding/neteq/tools/fake_decode_from_file.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/audio_decoder_proxy_factory.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { +namespace { + +constexpr int kSampleRateHz = 32000; +constexpr int kRunTimeMs = 10000; + +// This class implements a fake decoder. The decoder will read audio from a file +// and present as output, both for regular decoding and for PLC. +class AudioDecoderPlc : public AudioDecoder { + public: + AudioDecoderPlc(std::unique_ptr input, int sample_rate_hz) + : input_(std::move(input)), sample_rate_hz_(sample_rate_hz) {} + + void Reset() override {} + int SampleRateHz() const override { return sample_rate_hz_; } + size_t Channels() const override { return 1; } + int DecodeInternal(const uint8_t* /*encoded*/, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override { + RTC_CHECK_GE(encoded_len / 2, 10 * sample_rate_hz_ / 1000); + RTC_CHECK_LE(encoded_len / 2, 2 * 10 * sample_rate_hz_ / 1000); + RTC_CHECK_EQ(sample_rate_hz, sample_rate_hz_); + RTC_CHECK(decoded); + RTC_CHECK(speech_type); + RTC_CHECK(input_->Read(encoded_len / 2, decoded)); + *speech_type = kSpeech; + last_was_plc_ = false; + return encoded_len / 2; + } + + void GeneratePlc(size_t requested_samples_per_channel, + rtc::BufferT* concealment_audio) override { + // Instead of generating random data for GeneratePlc we use the same data as + // the input, so we can check that we produce the same result independently + // of the losses. + RTC_DCHECK_EQ(requested_samples_per_channel, 10 * sample_rate_hz_ / 1000); + + // Must keep a local copy of this since DecodeInternal sets it to false. + const bool last_was_plc = last_was_plc_; + + std::vector decoded(5760); + SpeechType speech_type; + int dec_len = DecodeInternal(nullptr, 2 * 10 * sample_rate_hz_ / 1000, + sample_rate_hz_, decoded.data(), &speech_type); + concealment_audio->AppendData(decoded.data(), dec_len); + concealed_samples_ += rtc::checked_cast(dec_len); + + if (!last_was_plc) { + ++concealment_events_; + } + last_was_plc_ = true; + } + + size_t concealed_samples() { return concealed_samples_; } + size_t concealment_events() { return concealment_events_; } + + private: + const std::unique_ptr input_; + const int sample_rate_hz_; + size_t concealed_samples_ = 0; + size_t concealment_events_ = 0; + bool last_was_plc_ = false; +}; + +// An input sample generator which generates only zero-samples. +class ZeroSampleGenerator : public EncodeNetEqInput::Generator { + public: + rtc::ArrayView Generate(size_t num_samples) override { + vec.resize(num_samples, 0); + rtc::ArrayView view(vec); + RTC_DCHECK_EQ(view.size(), num_samples); + return view; + } + + private: + std::vector vec; +}; + +// A NetEqInput which connects to another NetEqInput, but drops a number of +// consecutive packets on the way +class LossyInput : public NetEqInput { + public: + LossyInput(int loss_cadence, + int burst_length, + std::unique_ptr input) + : loss_cadence_(loss_cadence), + burst_length_(burst_length), + input_(std::move(input)) {} + + absl::optional NextPacketTime() const override { + return input_->NextPacketTime(); + } + + absl::optional NextOutputEventTime() const override { + return input_->NextOutputEventTime(); + } + + std::unique_ptr PopPacket() override { + if (loss_cadence_ != 0 && (++count_ % loss_cadence_) == 0) { + // Pop `burst_length_` packets to create the loss. + auto packet_to_return = input_->PopPacket(); + for (int i = 0; i < burst_length_; i++) { + input_->PopPacket(); + } + return packet_to_return; + } + return input_->PopPacket(); + } + + void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); } + + bool ended() const override { return input_->ended(); } + + absl::optional NextHeader() const override { + return input_->NextHeader(); + } + + private: + const int loss_cadence_; + const int burst_length_; + int count_ = 0; + const std::unique_ptr input_; +}; + +class AudioChecksumWithOutput : public AudioChecksum { + public: + explicit AudioChecksumWithOutput(std::string* output_str) + : output_str_(*output_str) {} + ~AudioChecksumWithOutput() { output_str_ = Finish(); } + + private: + std::string& output_str_; +}; + +struct TestStatistics { + NetEqNetworkStatistics network; + NetEqLifetimeStatistics lifetime; +}; + +TestStatistics RunTest(int loss_cadence, + int burst_length, + std::string* checksum) { + NetEq::Config config; + config.for_test_no_time_stretching = true; + + // The input is mostly useless. It sends zero-samples to a PCM16b encoder, + // but the actual encoded samples will never be used by the decoder in the + // test. See below about the decoder. + auto generator = std::make_unique(); + constexpr int kPayloadType = 100; + AudioEncoderPcm16B::Config encoder_config; + encoder_config.sample_rate_hz = kSampleRateHz; + encoder_config.payload_type = kPayloadType; + auto encoder = std::make_unique(encoder_config); + auto input = std::make_unique( + std::move(generator), std::move(encoder), kRunTimeMs); + // Wrap the input in a loss function. + auto lossy_input = std::make_unique(loss_cadence, burst_length, + std::move(input)); + + // Setting up decoders. + NetEqTest::DecoderMap decoders; + // Using a fake decoder which simply reads the output audio from a file. + auto input_file = std::make_unique( + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm")); + AudioDecoderPlc dec(std::move(input_file), kSampleRateHz); + // Masquerading as a PCM16b decoder. + decoders.emplace(kPayloadType, SdpAudioFormat("l16", 32000, 1)); + + // Output is simply a checksum calculator. + auto output = std::make_unique(checksum); + + // No callback objects. + NetEqTest::Callbacks callbacks; + + NetEqTest neteq_test( + config, /*decoder_factory=*/ + rtc::make_ref_counted(&dec), + /*codecs=*/decoders, /*text_log=*/nullptr, /*neteq_factory=*/nullptr, + /*input=*/std::move(lossy_input), std::move(output), callbacks); + EXPECT_LE(kRunTimeMs, neteq_test.Run()); + + auto lifetime_stats = neteq_test.LifetimeStats(); + EXPECT_EQ(dec.concealed_samples(), lifetime_stats.concealed_samples); + EXPECT_EQ(dec.concealment_events(), lifetime_stats.concealment_events); + return {neteq_test.SimulationStats(), neteq_test.LifetimeStats()}; +} +} // namespace + +// Check that some basic metrics are produced in the right direction. In +// particular, expand_rate should only increase if there are losses present. Our +// dummy decoder is designed such as the checksum should always be the same +// regardless of the losses given that calls are executed in the right order. +TEST(NetEqDecoderPlc, BasicMetrics) { + std::string checksum; + + // Drop 1 packet every 10 packets. + auto stats = RunTest(10, 1, &checksum); + + std::string checksum_no_loss; + auto stats_no_loss = RunTest(0, 0, &checksum_no_loss); + + EXPECT_EQ(checksum, checksum_no_loss); + + EXPECT_EQ(stats.network.preemptive_rate, + stats_no_loss.network.preemptive_rate); + EXPECT_EQ(stats.network.accelerate_rate, + stats_no_loss.network.accelerate_rate); + EXPECT_EQ(0, stats_no_loss.network.expand_rate); + EXPECT_GT(stats.network.expand_rate, 0); +} + +// Checks that interruptions are not counted in small losses but they are +// correctly counted in long interruptions. +TEST(NetEqDecoderPlc, CountInterruptions) { + std::string checksum; + std::string checksum_2; + std::string checksum_3; + + // Half of the packets lost but in short interruptions. + auto stats_no_interruptions = RunTest(1, 1, &checksum); + // One lost of 500 ms (250 packets). + auto stats_one_interruption = RunTest(200, 250, &checksum_2); + // Two losses of 250ms each (125 packets). + auto stats_two_interruptions = RunTest(125, 125, &checksum_3); + + EXPECT_EQ(checksum, checksum_2); + EXPECT_EQ(checksum, checksum_3); + EXPECT_GT(stats_no_interruptions.network.expand_rate, 0); + EXPECT_EQ(stats_no_interruptions.lifetime.total_interruption_duration_ms, 0); + EXPECT_EQ(stats_no_interruptions.lifetime.interruption_count, 0); + + EXPECT_GT(stats_one_interruption.network.expand_rate, 0); + EXPECT_EQ(stats_one_interruption.lifetime.total_interruption_duration_ms, + 5000); + EXPECT_EQ(stats_one_interruption.lifetime.interruption_count, 1); + + EXPECT_GT(stats_two_interruptions.network.expand_rate, 0); + EXPECT_EQ(stats_two_interruptions.lifetime.total_interruption_duration_ms, + 5000); + EXPECT_EQ(stats_two_interruptions.lifetime.interruption_count, 2); +} + +// Checks that small losses do not produce interruptions. +TEST(NetEqDecoderPlc, NoInterruptionsInSmallLosses) { + std::string checksum_1; + std::string checksum_4; + + auto stats_1 = RunTest(300, 1, &checksum_1); + auto stats_4 = RunTest(300, 4, &checksum_4); + + EXPECT_EQ(checksum_1, checksum_4); + + EXPECT_EQ(stats_1.lifetime.interruption_count, 0); + EXPECT_EQ(stats_1.lifetime.total_interruption_duration_ms, 0); + EXPECT_EQ(stats_1.lifetime.concealed_samples, 640u); // 20ms of concealment. + EXPECT_EQ(stats_1.lifetime.concealment_events, 1u); // in just one event. + + EXPECT_EQ(stats_4.lifetime.interruption_count, 0); + EXPECT_EQ(stats_4.lifetime.total_interruption_duration_ms, 0); + EXPECT_EQ(stats_4.lifetime.concealed_samples, 2560u); // 80ms of concealment. + EXPECT_EQ(stats_4.lifetime.concealment_events, 1u); // in just one event. +} + +// Checks that interruptions of different sizes report correct duration. +TEST(NetEqDecoderPlc, InterruptionsReportCorrectSize) { + std::string checksum; + + for (int burst_length = 5; burst_length < 10; burst_length++) { + auto stats = RunTest(300, burst_length, &checksum); + auto duration = stats.lifetime.total_interruption_duration_ms; + if (burst_length < 8) { + EXPECT_EQ(duration, 0); + } else { + EXPECT_EQ(duration, burst_length * 20); + } + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc new file mode 100644 index 0000000000..6a6367d045 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -0,0 +1,2141 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/neteq_impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "api/neteq/tick_timer.h" +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/codecs/cng/webrtc_cng.h" +#include "modules/audio_coding/neteq/accelerate.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/comfort_noise.h" +#include "modules/audio_coding/neteq/decision_logic.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/dtmf_buffer.h" +#include "modules/audio_coding/neteq/dtmf_tone_generator.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/merge.h" +#include "modules/audio_coding/neteq/nack_tracker.h" +#include "modules/audio_coding/neteq/normal.h" +#include "modules/audio_coding/neteq/packet.h" +#include "modules/audio_coding/neteq/packet_buffer.h" +#include "modules/audio_coding/neteq/post_decode_vad.h" +#include "modules/audio_coding/neteq/preemptive_expand.h" +#include "modules/audio_coding/neteq/red_payload_splitter.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "modules/audio_coding/neteq/time_stretch.h" +#include "modules/audio_coding/neteq/timestamp_scaler.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/sanitizer.h" +#include "rtc_base/strings/audio_format_to_string.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace { + +std::unique_ptr CreateNetEqController( + const NetEqControllerFactory& controller_factory, + int base_min_delay, + int max_packets_in_buffer, + bool allow_time_stretching, + TickTimer* tick_timer, + webrtc::Clock* clock) { + NetEqController::Config config; + config.base_min_delay_ms = base_min_delay; + config.max_packets_in_buffer = max_packets_in_buffer; + config.allow_time_stretching = allow_time_stretching; + config.tick_timer = tick_timer; + config.clock = clock; + return controller_factory.CreateNetEqController(config); +} + +} // namespace + +NetEqImpl::Dependencies::Dependencies( + const NetEq::Config& config, + Clock* clock, + const rtc::scoped_refptr& decoder_factory, + const NetEqControllerFactory& controller_factory) + : clock(clock), + tick_timer(new TickTimer), + stats(new StatisticsCalculator), + decoder_database( + new DecoderDatabase(decoder_factory, config.codec_pair_id)), + dtmf_buffer(new DtmfBuffer(config.sample_rate_hz)), + dtmf_tone_generator(new DtmfToneGenerator), + packet_buffer( + new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())), + neteq_controller( + CreateNetEqController(controller_factory, + config.min_delay_ms, + config.max_packets_in_buffer, + !config.for_test_no_time_stretching, + tick_timer.get(), + clock)), + red_payload_splitter(new RedPayloadSplitter), + timestamp_scaler(new TimestampScaler(*decoder_database)), + accelerate_factory(new AccelerateFactory), + expand_factory(new ExpandFactory), + preemptive_expand_factory(new PreemptiveExpandFactory) {} + +NetEqImpl::Dependencies::~Dependencies() = default; + +NetEqImpl::NetEqImpl(const NetEq::Config& config, + Dependencies&& deps, + bool create_components) + : clock_(deps.clock), + tick_timer_(std::move(deps.tick_timer)), + decoder_database_(std::move(deps.decoder_database)), + dtmf_buffer_(std::move(deps.dtmf_buffer)), + dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)), + packet_buffer_(std::move(deps.packet_buffer)), + red_payload_splitter_(std::move(deps.red_payload_splitter)), + timestamp_scaler_(std::move(deps.timestamp_scaler)), + vad_(new PostDecodeVad()), + expand_factory_(std::move(deps.expand_factory)), + accelerate_factory_(std::move(deps.accelerate_factory)), + preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)), + stats_(std::move(deps.stats)), + controller_(std::move(deps.neteq_controller)), + last_mode_(Mode::kNormal), + decoded_buffer_length_(kMaxFrameSize), + decoded_buffer_(new int16_t[decoded_buffer_length_]), + playout_timestamp_(0), + new_codec_(false), + timestamp_(0), + reset_decoder_(false), + first_packet_(true), + enable_fast_accelerate_(config.enable_fast_accelerate), + nack_enabled_(false), + enable_muted_state_(config.enable_muted_state), + expand_uma_logger_("WebRTC.Audio.ExpandRatePercent", + 10, // Report once every 10 s. + tick_timer_.get()), + speech_expand_uma_logger_("WebRTC.Audio.SpeechExpandRatePercent", + 10, // Report once every 10 s. + tick_timer_.get()), + no_time_stretching_(config.for_test_no_time_stretching) { + RTC_LOG(LS_INFO) << "NetEq config: " << config.ToString(); + int fs = config.sample_rate_hz; + if (fs != 8000 && fs != 16000 && fs != 32000 && fs != 48000) { + RTC_LOG(LS_ERROR) << "Sample rate " << fs + << " Hz not supported. " + "Changing to 8000 Hz."; + fs = 8000; + } + controller_->SetMaximumDelay(config.max_delay_ms); + fs_hz_ = fs; + fs_mult_ = fs / 8000; + last_output_sample_rate_hz_ = fs; + output_size_samples_ = static_cast(kOutputSizeMs * 8 * fs_mult_); + controller_->SetSampleRate(fs_hz_, output_size_samples_); + decoder_frame_length_ = 2 * output_size_samples_; // 20 ms. + if (create_components) { + SetSampleRateAndChannels(fs, 1); // Default is 1 channel. + } + RTC_DCHECK(!vad_->enabled()); + if (config.enable_post_decode_vad) { + vad_->Enable(); + } +} + +NetEqImpl::~NetEqImpl() = default; + +int NetEqImpl::InsertPacket(const RTPHeader& rtp_header, + rtc::ArrayView payload) { + rtc::MsanCheckInitialized(payload); + TRACE_EVENT0("webrtc", "NetEqImpl::InsertPacket"); + MutexLock lock(&mutex_); + if (InsertPacketInternal(rtp_header, payload) != 0) { + return kFail; + } + return kOK; +} + +void NetEqImpl::InsertEmptyPacket(const RTPHeader& rtp_header) { + MutexLock lock(&mutex_); + if (nack_enabled_) { + nack_->UpdateLastReceivedPacket(rtp_header.sequenceNumber, + rtp_header.timestamp); + } + controller_->RegisterEmptyPacket(); +} + +namespace { +void SetAudioFrameActivityAndType(bool vad_enabled, + NetEqImpl::OutputType type, + AudioFrame::VADActivity last_vad_activity, + AudioFrame* audio_frame) { + switch (type) { + case NetEqImpl::OutputType::kNormalSpeech: { + audio_frame->speech_type_ = AudioFrame::kNormalSpeech; + audio_frame->vad_activity_ = AudioFrame::kVadActive; + break; + } + case NetEqImpl::OutputType::kVadPassive: { + // This should only be reached if the VAD is enabled. + RTC_DCHECK(vad_enabled); + audio_frame->speech_type_ = AudioFrame::kNormalSpeech; + audio_frame->vad_activity_ = AudioFrame::kVadPassive; + break; + } + case NetEqImpl::OutputType::kCNG: { + audio_frame->speech_type_ = AudioFrame::kCNG; + audio_frame->vad_activity_ = AudioFrame::kVadPassive; + break; + } + case NetEqImpl::OutputType::kPLC: { + audio_frame->speech_type_ = AudioFrame::kPLC; + audio_frame->vad_activity_ = last_vad_activity; + break; + } + case NetEqImpl::OutputType::kPLCCNG: { + audio_frame->speech_type_ = AudioFrame::kPLCCNG; + audio_frame->vad_activity_ = AudioFrame::kVadPassive; + break; + } + case NetEqImpl::OutputType::kCodecPLC: { + audio_frame->speech_type_ = AudioFrame::kCodecPLC; + audio_frame->vad_activity_ = last_vad_activity; + break; + } + default: + RTC_DCHECK_NOTREACHED(); + } + if (!vad_enabled) { + // Always set kVadUnknown when receive VAD is inactive. + audio_frame->vad_activity_ = AudioFrame::kVadUnknown; + } +} +} // namespace + +int NetEqImpl::GetAudio(AudioFrame* audio_frame, + bool* muted, + int* current_sample_rate_hz, + absl::optional action_override) { + TRACE_EVENT0("webrtc", "NetEqImpl::GetAudio"); + MutexLock lock(&mutex_); + if (GetAudioInternal(audio_frame, muted, action_override) != 0) { + return kFail; + } + RTC_DCHECK_EQ( + audio_frame->sample_rate_hz_, + rtc::dchecked_cast(audio_frame->samples_per_channel_ * 100)); + RTC_DCHECK_EQ(*muted, audio_frame->muted()); + SetAudioFrameActivityAndType(vad_->enabled(), LastOutputType(), + last_vad_activity_, audio_frame); + last_vad_activity_ = audio_frame->vad_activity_; + last_output_sample_rate_hz_ = audio_frame->sample_rate_hz_; + RTC_DCHECK(last_output_sample_rate_hz_ == 8000 || + last_output_sample_rate_hz_ == 16000 || + last_output_sample_rate_hz_ == 32000 || + last_output_sample_rate_hz_ == 48000) + << "Unexpected sample rate " << last_output_sample_rate_hz_; + + if (current_sample_rate_hz) { + *current_sample_rate_hz = last_output_sample_rate_hz_; + } + + return kOK; +} + +void NetEqImpl::SetCodecs(const std::map& codecs) { + MutexLock lock(&mutex_); + const std::vector changed_payload_types = + decoder_database_->SetCodecs(codecs); + for (const int pt : changed_payload_types) { + packet_buffer_->DiscardPacketsWithPayloadType(pt, stats_.get()); + } +} + +bool NetEqImpl::RegisterPayloadType(int rtp_payload_type, + const SdpAudioFormat& audio_format) { + RTC_LOG(LS_VERBOSE) << "NetEqImpl::RegisterPayloadType: payload type " + << rtp_payload_type << ", codec " + << rtc::ToString(audio_format); + MutexLock lock(&mutex_); + return decoder_database_->RegisterPayload(rtp_payload_type, audio_format) == + DecoderDatabase::kOK; +} + +int NetEqImpl::RemovePayloadType(uint8_t rtp_payload_type) { + MutexLock lock(&mutex_); + int ret = decoder_database_->Remove(rtp_payload_type); + if (ret == DecoderDatabase::kOK || ret == DecoderDatabase::kDecoderNotFound) { + packet_buffer_->DiscardPacketsWithPayloadType(rtp_payload_type, + stats_.get()); + return kOK; + } + return kFail; +} + +void NetEqImpl::RemoveAllPayloadTypes() { + MutexLock lock(&mutex_); + decoder_database_->RemoveAll(); +} + +bool NetEqImpl::SetMinimumDelay(int delay_ms) { + MutexLock lock(&mutex_); + if (delay_ms >= 0 && delay_ms <= 10000) { + RTC_DCHECK(controller_.get()); + return controller_->SetMinimumDelay(delay_ms); + } + return false; +} + +bool NetEqImpl::SetMaximumDelay(int delay_ms) { + MutexLock lock(&mutex_); + if (delay_ms >= 0 && delay_ms <= 10000) { + RTC_DCHECK(controller_.get()); + return controller_->SetMaximumDelay(delay_ms); + } + return false; +} + +bool NetEqImpl::SetBaseMinimumDelayMs(int delay_ms) { + MutexLock lock(&mutex_); + if (delay_ms >= 0 && delay_ms <= 10000) { + return controller_->SetBaseMinimumDelay(delay_ms); + } + return false; +} + +int NetEqImpl::GetBaseMinimumDelayMs() const { + MutexLock lock(&mutex_); + return controller_->GetBaseMinimumDelay(); +} + +int NetEqImpl::TargetDelayMs() const { + MutexLock lock(&mutex_); + RTC_DCHECK(controller_.get()); + return controller_->TargetLevelMs(); +} + +int NetEqImpl::FilteredCurrentDelayMs() const { + MutexLock lock(&mutex_); + // Sum up the filtered packet buffer level with the future length of the sync + // buffer. + const int delay_samples = + controller_->GetFilteredBufferLevel() + sync_buffer_->FutureLength(); + // The division below will truncate. The return value is in ms. + return delay_samples / rtc::CheckedDivExact(fs_hz_, 1000); +} + +int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) { + MutexLock lock(&mutex_); + RTC_DCHECK(decoder_database_.get()); + *stats = CurrentNetworkStatisticsInternal(); + stats_->GetNetworkStatistics(decoder_frame_length_, stats); + return 0; +} + +NetEqNetworkStatistics NetEqImpl::CurrentNetworkStatistics() const { + MutexLock lock(&mutex_); + return CurrentNetworkStatisticsInternal(); +} + +NetEqNetworkStatistics NetEqImpl::CurrentNetworkStatisticsInternal() const { + RTC_DCHECK(decoder_database_.get()); + NetEqNetworkStatistics stats; + const size_t total_samples_in_buffers = + packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) + + sync_buffer_->FutureLength(); + + RTC_DCHECK(controller_.get()); + stats.preferred_buffer_size_ms = controller_->TargetLevelMs(); + stats.jitter_peaks_found = controller_->PeakFound(); + RTC_DCHECK_GT(fs_hz_, 0); + stats.current_buffer_size_ms = + static_cast(total_samples_in_buffers * 1000 / fs_hz_); + return stats; +} + +NetEqLifetimeStatistics NetEqImpl::GetLifetimeStatistics() const { + MutexLock lock(&mutex_); + return stats_->GetLifetimeStatistics(); +} + +NetEqOperationsAndState NetEqImpl::GetOperationsAndState() const { + MutexLock lock(&mutex_); + auto result = stats_->GetOperationsAndState(); + result.current_buffer_size_ms = + (packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) + + sync_buffer_->FutureLength()) * + 1000 / fs_hz_; + result.current_frame_size_ms = decoder_frame_length_ * 1000 / fs_hz_; + result.next_packet_available = packet_buffer_->PeekNextPacket() && + packet_buffer_->PeekNextPacket()->timestamp == + sync_buffer_->end_timestamp(); + return result; +} + +void NetEqImpl::EnableVad() { + MutexLock lock(&mutex_); + RTC_DCHECK(vad_.get()); + vad_->Enable(); +} + +void NetEqImpl::DisableVad() { + MutexLock lock(&mutex_); + RTC_DCHECK(vad_.get()); + vad_->Disable(); +} + +absl::optional NetEqImpl::GetPlayoutTimestamp() const { + MutexLock lock(&mutex_); + if (first_packet_ || last_mode_ == Mode::kRfc3389Cng || + last_mode_ == Mode::kCodecInternalCng) { + // We don't have a valid RTP timestamp until we have decoded our first + // RTP packet. Also, the RTP timestamp is not accurate while playing CNG, + // which is indicated by returning an empty value. + return absl::nullopt; + } + return timestamp_scaler_->ToExternal(playout_timestamp_); +} + +int NetEqImpl::last_output_sample_rate_hz() const { + MutexLock lock(&mutex_); + return last_output_sample_rate_hz_; +} + +absl::optional NetEqImpl::GetDecoderFormat( + int payload_type) const { + MutexLock lock(&mutex_); + const DecoderDatabase::DecoderInfo* const di = + decoder_database_->GetDecoderInfo(payload_type); + if (di) { + const AudioDecoder* const decoder = di->GetDecoder(); + // TODO(kwiberg): Why the special case for RED? + return DecoderFormat{ + /*sample_rate_hz=*/di->IsRed() ? 8000 : di->SampleRateHz(), + /*num_channels=*/ + decoder ? rtc::dchecked_cast(decoder->Channels()) : 1, + /*sdp_format=*/di->GetFormat()}; + } else { + // Payload type not registered. + return absl::nullopt; + } +} + +void NetEqImpl::FlushBuffers() { + MutexLock lock(&mutex_); + RTC_LOG(LS_VERBOSE) << "FlushBuffers"; + packet_buffer_->Flush(stats_.get()); + RTC_DCHECK(sync_buffer_.get()); + RTC_DCHECK(expand_.get()); + sync_buffer_->Flush(); + sync_buffer_->set_next_index(sync_buffer_->next_index() - + expand_->overlap_length()); + // Set to wait for new codec. + first_packet_ = true; +} + +void NetEqImpl::EnableNack(size_t max_nack_list_size) { + MutexLock lock(&mutex_); + if (!nack_enabled_) { + nack_ = std::make_unique(); + nack_enabled_ = true; + nack_->UpdateSampleRate(fs_hz_); + } + nack_->SetMaxNackListSize(max_nack_list_size); +} + +void NetEqImpl::DisableNack() { + MutexLock lock(&mutex_); + nack_.reset(); + nack_enabled_ = false; +} + +std::vector NetEqImpl::GetNackList(int64_t round_trip_time_ms) const { + MutexLock lock(&mutex_); + if (!nack_enabled_) { + return std::vector(); + } + RTC_DCHECK(nack_.get()); + return nack_->GetNackList(round_trip_time_ms); +} + +int NetEqImpl::SyncBufferSizeMs() const { + MutexLock lock(&mutex_); + return rtc::dchecked_cast(sync_buffer_->FutureLength() / + rtc::CheckedDivExact(fs_hz_, 1000)); +} + +const SyncBuffer* NetEqImpl::sync_buffer_for_test() const { + MutexLock lock(&mutex_); + return sync_buffer_.get(); +} + +NetEq::Operation NetEqImpl::last_operation_for_test() const { + MutexLock lock(&mutex_); + return last_operation_; +} + +// Methods below this line are private. + +int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header, + rtc::ArrayView payload) { + if (payload.empty()) { + RTC_LOG_F(LS_ERROR) << "payload is empty"; + return kInvalidPointer; + } + + Timestamp receive_time = clock_->CurrentTime(); + stats_->ReceivedPacket(); + + PacketList packet_list; + // Insert packet in a packet list. + packet_list.push_back([&rtp_header, &payload, &receive_time] { + // Convert to Packet. + Packet packet; + packet.payload_type = rtp_header.payloadType; + packet.sequence_number = rtp_header.sequenceNumber; + packet.timestamp = rtp_header.timestamp; + packet.payload.SetData(payload.data(), payload.size()); + packet.packet_info = RtpPacketInfo(rtp_header, receive_time); + // Waiting time will be set upon inserting the packet in the buffer. + RTC_DCHECK(!packet.waiting_time); + return packet; + }()); + + bool update_sample_rate_and_channels = first_packet_; + + if (update_sample_rate_and_channels) { + // Reset timestamp scaling. + timestamp_scaler_->Reset(); + } + + if (!decoder_database_->IsRed(rtp_header.payloadType)) { + // Scale timestamp to internal domain (only for some codecs). + timestamp_scaler_->ToInternal(&packet_list); + } + + // Store these for later use, since the first packet may very well disappear + // before we need these values. + uint32_t main_timestamp = packet_list.front().timestamp; + uint8_t main_payload_type = packet_list.front().payload_type; + uint16_t main_sequence_number = packet_list.front().sequence_number; + + // Reinitialize NetEq if it's needed (changed SSRC or first call). + if (update_sample_rate_and_channels) { + // Note: `first_packet_` will be cleared further down in this method, once + // the packet has been successfully inserted into the packet buffer. + + // Flush the packet buffer and DTMF buffer. + packet_buffer_->Flush(stats_.get()); + dtmf_buffer_->Flush(); + + // Update audio buffer timestamp. + sync_buffer_->IncreaseEndTimestamp(main_timestamp - timestamp_); + + // Update codecs. + timestamp_ = main_timestamp; + } + + if (nack_enabled_) { + RTC_DCHECK(nack_); + if (update_sample_rate_and_channels) { + nack_->Reset(); + } + nack_->UpdateLastReceivedPacket(main_sequence_number, main_timestamp); + } + + // Check for RED payload type, and separate payloads into several packets. + if (decoder_database_->IsRed(rtp_header.payloadType)) { + if (!red_payload_splitter_->SplitRed(&packet_list)) { + return kRedundancySplitError; + } + // Only accept a few RED payloads of the same type as the main data, + // DTMF events and CNG. + red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_); + if (packet_list.empty()) { + return kRedundancySplitError; + } + } + + // Check payload types. + if (decoder_database_->CheckPayloadTypes(packet_list) == + DecoderDatabase::kDecoderNotFound) { + return kUnknownRtpPayloadType; + } + + RTC_DCHECK(!packet_list.empty()); + + // Update main_timestamp, if new packets appear in the list + // after RED splitting. + if (decoder_database_->IsRed(rtp_header.payloadType)) { + timestamp_scaler_->ToInternal(&packet_list); + main_timestamp = packet_list.front().timestamp; + main_payload_type = packet_list.front().payload_type; + main_sequence_number = packet_list.front().sequence_number; + } + + // Process DTMF payloads. Cycle through the list of packets, and pick out any + // DTMF payloads found. + PacketList::iterator it = packet_list.begin(); + while (it != packet_list.end()) { + const Packet& current_packet = (*it); + RTC_DCHECK(!current_packet.payload.empty()); + if (decoder_database_->IsDtmf(current_packet.payload_type)) { + DtmfEvent event; + int ret = DtmfBuffer::ParseEvent(current_packet.timestamp, + current_packet.payload.data(), + current_packet.payload.size(), &event); + if (ret != DtmfBuffer::kOK) { + return kDtmfParsingError; + } + if (dtmf_buffer_->InsertEvent(event) != DtmfBuffer::kOK) { + return kDtmfInsertError; + } + it = packet_list.erase(it); + } else { + ++it; + } + } + + PacketList parsed_packet_list; + bool is_dtx = false; + while (!packet_list.empty()) { + Packet& packet = packet_list.front(); + const DecoderDatabase::DecoderInfo* info = + decoder_database_->GetDecoderInfo(packet.payload_type); + if (!info) { + RTC_LOG(LS_WARNING) << "SplitAudio unknown payload type"; + return kUnknownRtpPayloadType; + } + + if (info->IsComfortNoise()) { + // Carry comfort noise packets along. + parsed_packet_list.splice(parsed_packet_list.end(), packet_list, + packet_list.begin()); + } else { + const auto sequence_number = packet.sequence_number; + const auto payload_type = packet.payload_type; + const Packet::Priority original_priority = packet.priority; + const auto& packet_info = packet.packet_info; + auto packet_from_result = [&](AudioDecoder::ParseResult& result) { + Packet new_packet; + new_packet.sequence_number = sequence_number; + new_packet.payload_type = payload_type; + new_packet.timestamp = result.timestamp; + new_packet.priority.codec_level = result.priority; + new_packet.priority.red_level = original_priority.red_level; + new_packet.packet_info = packet_info; + new_packet.frame = std::move(result.frame); + return new_packet; + }; + + std::vector results = + info->GetDecoder()->ParsePayload(std::move(packet.payload), + packet.timestamp); + if (results.empty()) { + packet_list.pop_front(); + } else { + bool first = true; + for (auto& result : results) { + RTC_DCHECK(result.frame); + RTC_DCHECK_GE(result.priority, 0); + is_dtx = is_dtx || result.frame->IsDtxPacket(); + if (first) { + // Re-use the node and move it to parsed_packet_list. + packet_list.front() = packet_from_result(result); + parsed_packet_list.splice(parsed_packet_list.end(), packet_list, + packet_list.begin()); + first = false; + } else { + parsed_packet_list.push_back(packet_from_result(result)); + } + } + } + } + } + + // Calculate the number of primary (non-FEC/RED) packets. + const size_t number_of_primary_packets = std::count_if( + parsed_packet_list.begin(), parsed_packet_list.end(), + [](const Packet& in) { return in.priority.codec_level == 0; }); + if (number_of_primary_packets < parsed_packet_list.size()) { + stats_->SecondaryPacketsReceived(parsed_packet_list.size() - + number_of_primary_packets); + } + + // Insert packets in buffer. + const int target_level_ms = controller_->TargetLevelMs(); + const int ret = packet_buffer_->InsertPacketList( + &parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_, + ¤t_cng_rtp_payload_type_, stats_.get(), decoder_frame_length_, + last_output_sample_rate_hz_, target_level_ms); + bool buffer_flush_occured = false; + if (ret == PacketBuffer::kFlushed) { + // Reset DSP timestamp etc. if packet buffer flushed. + new_codec_ = true; + update_sample_rate_and_channels = true; + buffer_flush_occured = true; + } else if (ret == PacketBuffer::kPartialFlush) { + // Forward sync buffer timestamp + timestamp_ = packet_buffer_->PeekNextPacket()->timestamp; + sync_buffer_->IncreaseEndTimestamp(timestamp_ - + sync_buffer_->end_timestamp()); + buffer_flush_occured = true; + } else if (ret != PacketBuffer::kOK) { + return kOtherError; + } + + if (first_packet_) { + first_packet_ = false; + // Update the codec on the next GetAudio call. + new_codec_ = true; + } + + if (current_rtp_payload_type_) { + RTC_DCHECK(decoder_database_->GetDecoderInfo(*current_rtp_payload_type_)) + << "Payload type " << static_cast(*current_rtp_payload_type_) + << " is unknown where it shouldn't be"; + } + + if (update_sample_rate_and_channels && !packet_buffer_->Empty()) { + // We do not use `current_rtp_payload_type_` to |set payload_type|, but + // get the next RTP header from `packet_buffer_` to obtain the payload type. + // The reason for it is the following corner case. If NetEq receives a + // CNG packet with a sample rate different than the current CNG then it + // flushes its buffer, assuming send codec must have been changed. However, + // payload type of the hypothetically new send codec is not known. + const Packet* next_packet = packet_buffer_->PeekNextPacket(); + RTC_DCHECK(next_packet); + const int payload_type = next_packet->payload_type; + size_t channels = 1; + if (!decoder_database_->IsComfortNoise(payload_type)) { + AudioDecoder* decoder = decoder_database_->GetDecoder(payload_type); + RTC_DCHECK(decoder); // Payloads are already checked to be valid. + channels = decoder->Channels(); + } + const DecoderDatabase::DecoderInfo* decoder_info = + decoder_database_->GetDecoderInfo(payload_type); + RTC_DCHECK(decoder_info); + if (decoder_info->SampleRateHz() != fs_hz_ || + channels != algorithm_buffer_->Channels()) { + SetSampleRateAndChannels(decoder_info->SampleRateHz(), channels); + } + if (nack_enabled_) { + RTC_DCHECK(nack_); + // Update the sample rate even if the rate is not new, because of Reset(). + nack_->UpdateSampleRate(fs_hz_); + } + } + + const DecoderDatabase::DecoderInfo* dec_info = + decoder_database_->GetDecoderInfo(main_payload_type); + RTC_DCHECK(dec_info); // Already checked that the payload type is known. + + NetEqController::PacketArrivedInfo info; + info.is_cng_or_dtmf = dec_info->IsComfortNoise() || dec_info->IsDtmf(); + info.packet_length_samples = + number_of_primary_packets * decoder_frame_length_; + info.main_timestamp = main_timestamp; + info.main_sequence_number = main_sequence_number; + info.is_dtx = is_dtx; + info.buffer_flush = buffer_flush_occured; + + const bool should_update_stats = !new_codec_; + auto relative_delay = + controller_->PacketArrived(fs_hz_, should_update_stats, info); + if (relative_delay) { + stats_->RelativePacketArrivalDelay(relative_delay.value()); + } + return 0; +} + +int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame, + bool* muted, + absl::optional action_override) { + PacketList packet_list; + DtmfEvent dtmf_event; + Operation operation; + bool play_dtmf; + *muted = false; + last_decoded_packet_infos_.clear(); + tick_timer_->Increment(); + stats_->IncreaseCounter(output_size_samples_, fs_hz_); + const auto lifetime_stats = stats_->GetLifetimeStatistics(); + expand_uma_logger_.UpdateSampleCounter(lifetime_stats.concealed_samples, + fs_hz_); + speech_expand_uma_logger_.UpdateSampleCounter( + lifetime_stats.concealed_samples - + lifetime_stats.silent_concealed_samples, + fs_hz_); + + // Check for muted state. + if (enable_muted_state_ && expand_->Muted() && packet_buffer_->Empty()) { + RTC_DCHECK_EQ(last_mode_, Mode::kExpand); + audio_frame->Reset(); + RTC_DCHECK(audio_frame->muted()); // Reset() should mute the frame. + playout_timestamp_ += static_cast(output_size_samples_); + audio_frame->sample_rate_hz_ = fs_hz_; + // Make sure the total number of samples fits in the AudioFrame. + if (output_size_samples_ * sync_buffer_->Channels() > + AudioFrame::kMaxDataSizeSamples) { + return kSampleUnderrun; + } + audio_frame->samples_per_channel_ = output_size_samples_; + audio_frame->timestamp_ = + first_packet_ + ? 0 + : timestamp_scaler_->ToExternal(playout_timestamp_) - + static_cast(audio_frame->samples_per_channel_); + audio_frame->num_channels_ = sync_buffer_->Channels(); + stats_->ExpandedNoiseSamples(output_size_samples_, false); + controller_->NotifyMutedState(); + *muted = true; + return 0; + } + int return_value = GetDecision(&operation, &packet_list, &dtmf_event, + &play_dtmf, action_override); + if (return_value != 0) { + last_mode_ = Mode::kError; + return return_value; + } + + AudioDecoder::SpeechType speech_type; + int length = 0; + const size_t start_num_packets = packet_list.size(); + int decode_return_value = + Decode(&packet_list, &operation, &length, &speech_type); + + RTC_DCHECK(vad_.get()); + bool sid_frame_available = + (operation == Operation::kRfc3389Cng && !packet_list.empty()); + vad_->Update(decoded_buffer_.get(), static_cast(length), speech_type, + sid_frame_available, fs_hz_); + + // This is the criterion that we did decode some data through the speech + // decoder, and the operation resulted in comfort noise. + const bool codec_internal_sid_frame = + (speech_type == AudioDecoder::kComfortNoise && + start_num_packets > packet_list.size()); + + if (sid_frame_available || codec_internal_sid_frame) { + // Start a new stopwatch since we are decoding a new CNG packet. + generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch(); + } + + algorithm_buffer_->Clear(); + switch (operation) { + case Operation::kNormal: { + DoNormal(decoded_buffer_.get(), length, speech_type, play_dtmf); + if (length > 0) { + stats_->DecodedOutputPlayed(); + } + break; + } + case Operation::kMerge: { + DoMerge(decoded_buffer_.get(), length, speech_type, play_dtmf); + break; + } + case Operation::kExpand: { + RTC_DCHECK_EQ(return_value, 0); + if (!current_rtp_payload_type_ || !DoCodecPlc()) { + return_value = DoExpand(play_dtmf); + } + RTC_DCHECK_GE(sync_buffer_->FutureLength() - expand_->overlap_length(), + output_size_samples_); + break; + } + case Operation::kAccelerate: + case Operation::kFastAccelerate: { + const bool fast_accelerate = + enable_fast_accelerate_ && (operation == Operation::kFastAccelerate); + return_value = DoAccelerate(decoded_buffer_.get(), length, speech_type, + play_dtmf, fast_accelerate); + break; + } + case Operation::kPreemptiveExpand: { + return_value = DoPreemptiveExpand(decoded_buffer_.get(), length, + speech_type, play_dtmf); + break; + } + case Operation::kRfc3389Cng: + case Operation::kRfc3389CngNoPacket: { + return_value = DoRfc3389Cng(&packet_list, play_dtmf); + break; + } + case Operation::kCodecInternalCng: { + // This handles the case when there is no transmission and the decoder + // should produce internal comfort noise. + // TODO(hlundin): Write test for codec-internal CNG. + DoCodecInternalCng(decoded_buffer_.get(), length); + break; + } + case Operation::kDtmf: { + // TODO(hlundin): Write test for this. + return_value = DoDtmf(dtmf_event, &play_dtmf); + break; + } + case Operation::kUndefined: { + RTC_LOG(LS_ERROR) << "Invalid operation kUndefined."; + RTC_DCHECK_NOTREACHED(); // This should not happen. + last_mode_ = Mode::kError; + return kInvalidOperation; + } + } // End of switch. + last_operation_ = operation; + if (return_value < 0) { + return return_value; + } + + if (last_mode_ != Mode::kRfc3389Cng) { + comfort_noise_->Reset(); + } + + // We treat it as if all packets referenced to by `last_decoded_packet_infos_` + // were mashed together when creating the samples in `algorithm_buffer_`. + RtpPacketInfos packet_infos(last_decoded_packet_infos_); + + // Copy samples from `algorithm_buffer_` to `sync_buffer_`. + // + // TODO(bugs.webrtc.org/10757): + // We would in the future also like to pass `packet_infos` so that we can do + // sample-perfect tracking of that information across `sync_buffer_`. + sync_buffer_->PushBack(*algorithm_buffer_); + + // Extract data from `sync_buffer_` to `output`. + size_t num_output_samples_per_channel = output_size_samples_; + size_t num_output_samples = output_size_samples_ * sync_buffer_->Channels(); + if (num_output_samples > AudioFrame::kMaxDataSizeSamples) { + RTC_LOG(LS_WARNING) << "Output array is too short. " + << AudioFrame::kMaxDataSizeSamples << " < " + << output_size_samples_ << " * " + << sync_buffer_->Channels(); + num_output_samples = AudioFrame::kMaxDataSizeSamples; + num_output_samples_per_channel = + AudioFrame::kMaxDataSizeSamples / sync_buffer_->Channels(); + } + sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel, + audio_frame); + audio_frame->sample_rate_hz_ = fs_hz_; + // TODO(bugs.webrtc.org/10757): + // We don't have the ability to properly track individual packets once their + // audio samples have entered `sync_buffer_`. So for now, treat it as if + // `packet_infos` from packets decoded by the current `GetAudioInternal()` + // call were all consumed assembling the current audio frame and the current + // audio frame only. + audio_frame->packet_infos_ = std::move(packet_infos); + if (sync_buffer_->FutureLength() < expand_->overlap_length()) { + // The sync buffer should always contain `overlap_length` samples, but now + // too many samples have been extracted. Reinstall the `overlap_length` + // lookahead by moving the index. + const size_t missing_lookahead_samples = + expand_->overlap_length() - sync_buffer_->FutureLength(); + RTC_DCHECK_GE(sync_buffer_->next_index(), missing_lookahead_samples); + sync_buffer_->set_next_index(sync_buffer_->next_index() - + missing_lookahead_samples); + } + if (audio_frame->samples_per_channel_ != output_size_samples_) { + RTC_LOG(LS_ERROR) << "audio_frame->samples_per_channel_ (" + << audio_frame->samples_per_channel_ + << ") != output_size_samples_ (" << output_size_samples_ + << ")"; + // TODO(minyue): treatment of under-run, filling zeros + audio_frame->Mute(); + return kSampleUnderrun; + } + + // Should always have overlap samples left in the `sync_buffer_`. + RTC_DCHECK_GE(sync_buffer_->FutureLength(), expand_->overlap_length()); + + // TODO(yujo): For muted frames, this can be a copy rather than an addition. + if (play_dtmf) { + return_value = DtmfOverdub(dtmf_event, sync_buffer_->Channels(), + audio_frame->mutable_data()); + } + + // Update the background noise parameters if last operation wrote data + // straight from the decoder to the `sync_buffer_`. That is, none of the + // operations that modify the signal can be followed by a parameter update. + if ((last_mode_ == Mode::kNormal) || (last_mode_ == Mode::kAccelerateFail) || + (last_mode_ == Mode::kPreemptiveExpandFail) || + (last_mode_ == Mode::kRfc3389Cng) || + (last_mode_ == Mode::kCodecInternalCng)) { + background_noise_->Update(*sync_buffer_, *vad_.get()); + } + + if (operation == Operation::kDtmf) { + // DTMF data was written the end of `sync_buffer_`. + // Update index to end of DTMF data in `sync_buffer_`. + sync_buffer_->set_dtmf_index(sync_buffer_->Size()); + } + + if (last_mode_ != Mode::kExpand && last_mode_ != Mode::kCodecPlc) { + // If last operation was not expand, calculate the `playout_timestamp_` from + // the `sync_buffer_`. However, do not update the `playout_timestamp_` if it + // would be moved "backwards". + uint32_t temp_timestamp = + sync_buffer_->end_timestamp() - + static_cast(sync_buffer_->FutureLength()); + if (static_cast(temp_timestamp - playout_timestamp_) > 0) { + playout_timestamp_ = temp_timestamp; + } + } else { + // Use dead reckoning to estimate the `playout_timestamp_`. + playout_timestamp_ += static_cast(output_size_samples_); + } + // Set the timestamp in the audio frame to zero before the first packet has + // been inserted. Otherwise, subtract the frame size in samples to get the + // timestamp of the first sample in the frame (playout_timestamp_ is the + // last + 1). + audio_frame->timestamp_ = + first_packet_ + ? 0 + : timestamp_scaler_->ToExternal(playout_timestamp_) - + static_cast(audio_frame->samples_per_channel_); + + if (!(last_mode_ == Mode::kRfc3389Cng || + last_mode_ == Mode::kCodecInternalCng || last_mode_ == Mode::kExpand || + last_mode_ == Mode::kCodecPlc)) { + generated_noise_stopwatch_.reset(); + } + + if (decode_return_value) + return decode_return_value; + return return_value; +} + +int NetEqImpl::GetDecision(Operation* operation, + PacketList* packet_list, + DtmfEvent* dtmf_event, + bool* play_dtmf, + absl::optional action_override) { + // Initialize output variables. + *play_dtmf = false; + *operation = Operation::kUndefined; + + RTC_DCHECK(sync_buffer_.get()); + uint32_t end_timestamp = sync_buffer_->end_timestamp(); + if (!new_codec_) { + const uint32_t five_seconds_samples = 5 * fs_hz_; + packet_buffer_->DiscardOldPackets(end_timestamp, five_seconds_samples, + stats_.get()); + } + const Packet* packet = packet_buffer_->PeekNextPacket(); + + RTC_DCHECK(!generated_noise_stopwatch_ || + generated_noise_stopwatch_->ElapsedTicks() >= 1); + uint64_t generated_noise_samples = + generated_noise_stopwatch_ ? (generated_noise_stopwatch_->ElapsedTicks() - + 1) * output_size_samples_ + + controller_->noise_fast_forward() + : 0; + + if (controller_->CngRfc3389On() || last_mode_ == Mode::kRfc3389Cng) { + // Because of timestamp peculiarities, we have to "manually" disallow using + // a CNG packet with the same timestamp as the one that was last played. + // This can happen when using redundancy and will cause the timing to shift. + while (packet && decoder_database_->IsComfortNoise(packet->payload_type) && + (end_timestamp >= packet->timestamp || + end_timestamp + generated_noise_samples > packet->timestamp)) { + // Don't use this packet, discard it. + if (packet_buffer_->DiscardNextPacket(stats_.get()) != + PacketBuffer::kOK) { + RTC_DCHECK_NOTREACHED(); // Must be ok by design. + } + // Check buffer again. + if (!new_codec_) { + packet_buffer_->DiscardOldPackets(end_timestamp, 5 * fs_hz_, + stats_.get()); + } + packet = packet_buffer_->PeekNextPacket(); + } + } + + RTC_DCHECK(expand_.get()); + const int samples_left = static_cast(sync_buffer_->FutureLength() - + expand_->overlap_length()); + if (last_mode_ == Mode::kAccelerateSuccess || + last_mode_ == Mode::kAccelerateLowEnergy || + last_mode_ == Mode::kPreemptiveExpandSuccess || + last_mode_ == Mode::kPreemptiveExpandLowEnergy) { + // Subtract (samples_left + output_size_samples_) from sampleMemory. + controller_->AddSampleMemory( + -(samples_left + rtc::dchecked_cast(output_size_samples_))); + } + + // Check if it is time to play a DTMF event. + if (dtmf_buffer_->GetEvent( + static_cast(end_timestamp + generated_noise_samples), + dtmf_event)) { + *play_dtmf = true; + } + + // Get instruction. + RTC_DCHECK(sync_buffer_.get()); + RTC_DCHECK(expand_.get()); + generated_noise_samples = + generated_noise_stopwatch_ + ? generated_noise_stopwatch_->ElapsedTicks() * output_size_samples_ + + controller_->noise_fast_forward() + : 0; + NetEqController::NetEqStatus status; + status.packet_buffer_info.dtx_or_cng = + packet_buffer_->ContainsDtxOrCngPacket(decoder_database_.get()); + status.packet_buffer_info.num_samples = + packet_buffer_->NumSamplesInBuffer(decoder_frame_length_); + status.packet_buffer_info.span_samples = packet_buffer_->GetSpanSamples( + decoder_frame_length_, last_output_sample_rate_hz_, true); + status.packet_buffer_info.span_samples_no_dtx = + packet_buffer_->GetSpanSamples(decoder_frame_length_, + last_output_sample_rate_hz_, false); + status.packet_buffer_info.num_packets = packet_buffer_->NumPacketsInBuffer(); + status.target_timestamp = sync_buffer_->end_timestamp(); + status.expand_mutefactor = expand_->MuteFactor(0); + status.last_packet_samples = decoder_frame_length_; + status.last_mode = last_mode_; + status.play_dtmf = *play_dtmf; + status.generated_noise_samples = generated_noise_samples; + status.sync_buffer_samples = sync_buffer_->FutureLength(); + if (packet) { + status.next_packet = { + packet->timestamp, packet->frame && packet->frame->IsDtxPacket(), + decoder_database_->IsComfortNoise(packet->payload_type)}; + } + *operation = controller_->GetDecision(status, &reset_decoder_); + + // Disallow time stretching if this packet is DTX, because such a decision may + // be based on earlier buffer level estimate, as we do not update buffer level + // during DTX. When we have a better way to update buffer level during DTX, + // this can be discarded. + if (packet && packet->frame && packet->frame->IsDtxPacket() && + (*operation == Operation::kMerge || + *operation == Operation::kAccelerate || + *operation == Operation::kFastAccelerate || + *operation == Operation::kPreemptiveExpand)) { + *operation = Operation::kNormal; + } + + if (action_override) { + // Use the provided action instead of the decision NetEq decided on. + *operation = *action_override; + } + // Check if we already have enough samples in the `sync_buffer_`. If so, + // change decision to normal, unless the decision was merge, accelerate, or + // preemptive expand. + if (samples_left >= rtc::dchecked_cast(output_size_samples_) && + *operation != Operation::kMerge && *operation != Operation::kAccelerate && + *operation != Operation::kFastAccelerate && + *operation != Operation::kPreemptiveExpand) { + *operation = Operation::kNormal; + return 0; + } + + controller_->ExpandDecision(*operation); + if ((last_mode_ == Mode::kCodecPlc) && (*operation != Operation::kExpand)) { + // Getting out of the PLC expand mode, reporting interruptions. + // NetEq PLC reports this metrics in expand.cc + stats_->EndExpandEvent(fs_hz_); + } + + // Check conditions for reset. + if (new_codec_ || *operation == Operation::kUndefined) { + // The only valid reason to get kUndefined is that new_codec_ is set. + RTC_DCHECK(new_codec_); + if (*play_dtmf && !packet) { + timestamp_ = dtmf_event->timestamp; + } else { + if (!packet) { + RTC_LOG(LS_ERROR) << "Packet missing where it shouldn't."; + return -1; + } + timestamp_ = packet->timestamp; + if (*operation == Operation::kRfc3389CngNoPacket && + decoder_database_->IsComfortNoise(packet->payload_type)) { + // Change decision to CNG packet, since we do have a CNG packet, but it + // was considered too early to use. Now, use it anyway. + *operation = Operation::kRfc3389Cng; + } else if (*operation != Operation::kRfc3389Cng) { + *operation = Operation::kNormal; + } + } + // Adjust `sync_buffer_` timestamp before setting `end_timestamp` to the + // new value. + sync_buffer_->IncreaseEndTimestamp(timestamp_ - end_timestamp); + end_timestamp = timestamp_; + new_codec_ = false; + controller_->SoftReset(); + stats_->ResetMcu(); + } + + size_t required_samples = output_size_samples_; + const size_t samples_10_ms = static_cast(80 * fs_mult_); + const size_t samples_20_ms = 2 * samples_10_ms; + const size_t samples_30_ms = 3 * samples_10_ms; + + switch (*operation) { + case Operation::kExpand: { + timestamp_ = end_timestamp; + return 0; + } + case Operation::kRfc3389CngNoPacket: + case Operation::kCodecInternalCng: { + return 0; + } + case Operation::kDtmf: { + // TODO(hlundin): Write test for this. + // Update timestamp. + timestamp_ = end_timestamp; + const uint64_t generated_noise_samples = + generated_noise_stopwatch_ + ? generated_noise_stopwatch_->ElapsedTicks() * + output_size_samples_ + + controller_->noise_fast_forward() + : 0; + if (generated_noise_samples > 0 && last_mode_ != Mode::kDtmf) { + // Make a jump in timestamp due to the recently played comfort noise. + uint32_t timestamp_jump = + static_cast(generated_noise_samples); + sync_buffer_->IncreaseEndTimestamp(timestamp_jump); + timestamp_ += timestamp_jump; + } + return 0; + } + case Operation::kAccelerate: + case Operation::kFastAccelerate: { + // In order to do an accelerate we need at least 30 ms of audio data. + if (samples_left >= static_cast(samples_30_ms)) { + // Already have enough data, so we do not need to extract any more. + controller_->set_sample_memory(samples_left); + controller_->set_prev_time_scale(true); + return 0; + } else if (samples_left >= static_cast(samples_10_ms) && + decoder_frame_length_ >= samples_30_ms) { + // Avoid decoding more data as it might overflow the playout buffer. + *operation = Operation::kNormal; + return 0; + } else if (samples_left < static_cast(samples_20_ms) && + decoder_frame_length_ < samples_30_ms) { + // Build up decoded data by decoding at least 20 ms of audio data. Do + // not perform accelerate yet, but wait until we only need to do one + // decoding. + required_samples = 2 * output_size_samples_; + *operation = Operation::kNormal; + } + // If none of the above is true, we have one of two possible situations: + // (1) 20 ms <= samples_left < 30 ms and decoder_frame_length_ < 30 ms; or + // (2) samples_left < 10 ms and decoder_frame_length_ >= 30 ms. + // In either case, we move on with the accelerate decision, and decode one + // frame now. + break; + } + case Operation::kPreemptiveExpand: { + // In order to do a preemptive expand we need at least 30 ms of decoded + // audio data. + if ((samples_left >= static_cast(samples_30_ms)) || + (samples_left >= static_cast(samples_10_ms) && + decoder_frame_length_ >= samples_30_ms)) { + // Already have enough data, so we do not need to extract any more. + // Or, avoid decoding more data as it might overflow the playout buffer. + // Still try preemptive expand, though. + controller_->set_sample_memory(samples_left); + controller_->set_prev_time_scale(true); + return 0; + } + if (samples_left < static_cast(samples_20_ms) && + decoder_frame_length_ < samples_30_ms) { + // Build up decoded data by decoding at least 20 ms of audio data. + // Still try to perform preemptive expand. + required_samples = 2 * output_size_samples_; + } + // Move on with the preemptive expand decision. + break; + } + case Operation::kMerge: { + required_samples = + std::max(merge_->RequiredFutureSamples(), required_samples); + break; + } + default: { + // Do nothing. + } + } + + // Get packets from buffer. + int extracted_samples = 0; + if (packet) { + sync_buffer_->IncreaseEndTimestamp(packet->timestamp - end_timestamp); + + if (*operation != Operation::kRfc3389Cng) { + // We are about to decode and use a non-CNG packet. + controller_->SetCngOff(); + } + + extracted_samples = ExtractPackets(required_samples, packet_list); + if (extracted_samples < 0) { + return kPacketBufferCorruption; + } + } + + if (*operation == Operation::kAccelerate || + *operation == Operation::kFastAccelerate || + *operation == Operation::kPreemptiveExpand) { + controller_->set_sample_memory(samples_left + extracted_samples); + controller_->set_prev_time_scale(true); + } + + if (*operation == Operation::kAccelerate || + *operation == Operation::kFastAccelerate) { + // Check that we have enough data (30ms) to do accelerate. + if (extracted_samples + samples_left < static_cast(samples_30_ms)) { + // TODO(hlundin): Write test for this. + // Not enough, do normal operation instead. + *operation = Operation::kNormal; + } + } + + timestamp_ = sync_buffer_->end_timestamp(); + return 0; +} + +int NetEqImpl::Decode(PacketList* packet_list, + Operation* operation, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) { + *speech_type = AudioDecoder::kSpeech; + + // When packet_list is empty, we may be in kCodecInternalCng mode, and for + // that we use current active decoder. + AudioDecoder* decoder = decoder_database_->GetActiveDecoder(); + + if (!packet_list->empty()) { + const Packet& packet = packet_list->front(); + uint8_t payload_type = packet.payload_type; + if (!decoder_database_->IsComfortNoise(payload_type)) { + decoder = decoder_database_->GetDecoder(payload_type); + RTC_DCHECK(decoder); + if (!decoder) { + RTC_LOG(LS_WARNING) + << "Unknown payload type " << static_cast(payload_type); + packet_list->clear(); + return kDecoderNotFound; + } + bool decoder_changed; + decoder_database_->SetActiveDecoder(payload_type, &decoder_changed); + if (decoder_changed) { + // We have a new decoder. Re-init some values. + const DecoderDatabase::DecoderInfo* decoder_info = + decoder_database_->GetDecoderInfo(payload_type); + RTC_DCHECK(decoder_info); + if (!decoder_info) { + RTC_LOG(LS_WARNING) + << "Unknown payload type " << static_cast(payload_type); + packet_list->clear(); + return kDecoderNotFound; + } + // If sampling rate or number of channels has changed, we need to make + // a reset. + if (decoder_info->SampleRateHz() != fs_hz_ || + decoder->Channels() != algorithm_buffer_->Channels()) { + // TODO(tlegrand): Add unittest to cover this event. + SetSampleRateAndChannels(decoder_info->SampleRateHz(), + decoder->Channels()); + } + sync_buffer_->set_end_timestamp(timestamp_); + playout_timestamp_ = timestamp_; + } + } + } + + if (reset_decoder_) { + // TODO(hlundin): Write test for this. + if (decoder) + decoder->Reset(); + + // Reset comfort noise decoder. + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + if (cng_decoder) + cng_decoder->Reset(); + + reset_decoder_ = false; + } + + *decoded_length = 0; + // Update codec-internal PLC state. + if ((*operation == Operation::kMerge) && decoder && decoder->HasDecodePlc()) { + decoder->DecodePlc(1, &decoded_buffer_[*decoded_length]); + } + + int return_value; + if (*operation == Operation::kCodecInternalCng) { + RTC_DCHECK(packet_list->empty()); + return_value = DecodeCng(decoder, decoded_length, speech_type); + } else { + return_value = DecodeLoop(packet_list, *operation, decoder, decoded_length, + speech_type); + } + + if (*decoded_length < 0) { + // Error returned from the decoder. + *decoded_length = 0; + sync_buffer_->IncreaseEndTimestamp( + static_cast(decoder_frame_length_)); + int error_code = 0; + if (decoder) + error_code = decoder->ErrorCode(); + if (error_code != 0) { + // Got some error code from the decoder. + return_value = kDecoderErrorCode; + RTC_LOG(LS_WARNING) << "Decoder returned error code: " << error_code; + } else { + // Decoder does not implement error codes. Return generic error. + return_value = kOtherDecoderError; + RTC_LOG(LS_WARNING) << "Decoder error (no error code)"; + } + *operation = Operation::kExpand; // Do expansion to get data instead. + } + if (*speech_type != AudioDecoder::kComfortNoise) { + // Don't increment timestamp if codec returned CNG speech type + // since in this case, the we will increment the CNGplayedTS counter. + // Increase with number of samples per channel. + RTC_DCHECK(*decoded_length == 0 || + (decoder && decoder->Channels() == sync_buffer_->Channels())); + sync_buffer_->IncreaseEndTimestamp( + *decoded_length / static_cast(sync_buffer_->Channels())); + } + return return_value; +} + +int NetEqImpl::DecodeCng(AudioDecoder* decoder, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) { + if (!decoder) { + // This happens when active decoder is not defined. + *decoded_length = -1; + return 0; + } + + while (*decoded_length < rtc::dchecked_cast(output_size_samples_)) { + const int length = decoder->Decode( + nullptr, 0, fs_hz_, + (decoded_buffer_length_ - *decoded_length) * sizeof(int16_t), + &decoded_buffer_[*decoded_length], speech_type); + if (length > 0) { + *decoded_length += length; + } else { + // Error. + RTC_LOG(LS_WARNING) << "Failed to decode CNG"; + *decoded_length = -1; + break; + } + if (*decoded_length > static_cast(decoded_buffer_length_)) { + // Guard against overflow. + RTC_LOG(LS_WARNING) << "Decoded too much CNG."; + return kDecodedTooMuch; + } + } + stats_->GeneratedNoiseSamples(*decoded_length); + return 0; +} + +int NetEqImpl::DecodeLoop(PacketList* packet_list, + const Operation& operation, + AudioDecoder* decoder, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) { + RTC_DCHECK(last_decoded_packet_infos_.empty()); + + // Do decoding. + while (!packet_list->empty() && !decoder_database_->IsComfortNoise( + packet_list->front().payload_type)) { + RTC_DCHECK(decoder); // At this point, we must have a decoder object. + // The number of channels in the `sync_buffer_` should be the same as the + // number decoder channels. + RTC_DCHECK_EQ(sync_buffer_->Channels(), decoder->Channels()); + RTC_DCHECK_GE(decoded_buffer_length_, kMaxFrameSize * decoder->Channels()); + RTC_DCHECK(operation == Operation::kNormal || + operation == Operation::kAccelerate || + operation == Operation::kFastAccelerate || + operation == Operation::kMerge || + operation == Operation::kPreemptiveExpand); + + auto opt_result = packet_list->front().frame->Decode( + rtc::ArrayView(&decoded_buffer_[*decoded_length], + decoded_buffer_length_ - *decoded_length)); + last_decoded_packet_infos_.push_back( + std::move(packet_list->front().packet_info)); + packet_list->pop_front(); + if (opt_result) { + const auto& result = *opt_result; + *speech_type = result.speech_type; + if (result.num_decoded_samples > 0) { + *decoded_length += rtc::dchecked_cast(result.num_decoded_samples); + // Update `decoder_frame_length_` with number of samples per channel. + decoder_frame_length_ = + result.num_decoded_samples / decoder->Channels(); + } + } else { + // Error. + // TODO(ossu): What to put here? + RTC_LOG(LS_WARNING) << "Decode error"; + *decoded_length = -1; + last_decoded_packet_infos_.clear(); + packet_list->clear(); + break; + } + if (*decoded_length > rtc::dchecked_cast(decoded_buffer_length_)) { + // Guard against overflow. + RTC_LOG(LS_WARNING) << "Decoded too much."; + packet_list->clear(); + return kDecodedTooMuch; + } + } // End of decode loop. + + // If the list is not empty at this point, either a decoding error terminated + // the while-loop, or list must hold exactly one CNG packet. + RTC_DCHECK( + packet_list->empty() || *decoded_length < 0 || + (packet_list->size() == 1 && + decoder_database_->IsComfortNoise(packet_list->front().payload_type))); + return 0; +} + +void NetEqImpl::DoNormal(const int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) { + RTC_DCHECK(normal_.get()); + normal_->Process(decoded_buffer, decoded_length, last_mode_, + algorithm_buffer_.get()); + if (decoded_length != 0) { + last_mode_ = Mode::kNormal; + } + + // If last packet was decoded as an inband CNG, set mode to CNG instead. + if ((speech_type == AudioDecoder::kComfortNoise) || + ((last_mode_ == Mode::kCodecInternalCng) && (decoded_length == 0))) { + // TODO(hlundin): Remove second part of || statement above. + last_mode_ = Mode::kCodecInternalCng; + } + + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } +} + +void NetEqImpl::DoMerge(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) { + RTC_DCHECK(merge_.get()); + size_t new_length = + merge_->Process(decoded_buffer, decoded_length, algorithm_buffer_.get()); + // Correction can be negative. + int expand_length_correction = + rtc::dchecked_cast(new_length) - + rtc::dchecked_cast(decoded_length / algorithm_buffer_->Channels()); + + // Update in-call and post-call statistics. + if (expand_->MuteFactor(0) == 0) { + // Expand generates only noise. + stats_->ExpandedNoiseSamplesCorrection(expand_length_correction); + } else { + // Expansion generates more than only noise. + stats_->ExpandedVoiceSamplesCorrection(expand_length_correction); + } + + last_mode_ = Mode::kMerge; + // If last packet was decoded as an inband CNG, set mode to CNG instead. + if (speech_type == AudioDecoder::kComfortNoise) { + last_mode_ = Mode::kCodecInternalCng; + } + expand_->Reset(); + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } +} + +bool NetEqImpl::DoCodecPlc() { + AudioDecoder* decoder = decoder_database_->GetActiveDecoder(); + if (!decoder) { + return false; + } + const size_t channels = algorithm_buffer_->Channels(); + const size_t requested_samples_per_channel = + output_size_samples_ - + (sync_buffer_->FutureLength() - expand_->overlap_length()); + concealment_audio_.Clear(); + decoder->GeneratePlc(requested_samples_per_channel, &concealment_audio_); + if (concealment_audio_.empty()) { + // Nothing produced. Resort to regular expand. + return false; + } + RTC_CHECK_GE(concealment_audio_.size(), + requested_samples_per_channel * channels); + sync_buffer_->PushBackInterleaved(concealment_audio_); + RTC_DCHECK_NE(algorithm_buffer_->Channels(), 0); + const size_t concealed_samples_per_channel = + concealment_audio_.size() / channels; + + // Update in-call and post-call statistics. + const bool is_new_concealment_event = (last_mode_ != Mode::kCodecPlc); + if (std::all_of(concealment_audio_.cbegin(), concealment_audio_.cend(), + [](int16_t i) { return i == 0; })) { + // Expand operation generates only noise. + stats_->ExpandedNoiseSamples(concealed_samples_per_channel, + is_new_concealment_event); + } else { + // Expand operation generates more than only noise. + stats_->ExpandedVoiceSamples(concealed_samples_per_channel, + is_new_concealment_event); + } + last_mode_ = Mode::kCodecPlc; + if (!generated_noise_stopwatch_) { + // Start a new stopwatch since we may be covering for a lost CNG packet. + generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch(); + } + return true; +} + +int NetEqImpl::DoExpand(bool play_dtmf) { + while ((sync_buffer_->FutureLength() - expand_->overlap_length()) < + output_size_samples_) { + algorithm_buffer_->Clear(); + int return_value = expand_->Process(algorithm_buffer_.get()); + size_t length = algorithm_buffer_->Size(); + bool is_new_concealment_event = (last_mode_ != Mode::kExpand); + + // Update in-call and post-call statistics. + if (expand_->MuteFactor(0) == 0) { + // Expand operation generates only noise. + stats_->ExpandedNoiseSamples(length, is_new_concealment_event); + } else { + // Expand operation generates more than only noise. + stats_->ExpandedVoiceSamples(length, is_new_concealment_event); + } + + last_mode_ = Mode::kExpand; + + if (return_value < 0) { + return return_value; + } + + sync_buffer_->PushBack(*algorithm_buffer_); + algorithm_buffer_->Clear(); + } + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } + + if (!generated_noise_stopwatch_) { + // Start a new stopwatch since we may be covering for a lost CNG packet. + generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch(); + } + + return 0; +} + +int NetEqImpl::DoAccelerate(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf, + bool fast_accelerate) { + const size_t required_samples = + static_cast(240 * fs_mult_); // Must have 30 ms. + size_t borrowed_samples_per_channel = 0; + size_t num_channels = algorithm_buffer_->Channels(); + size_t decoded_length_per_channel = decoded_length / num_channels; + if (decoded_length_per_channel < required_samples) { + // Must move data from the `sync_buffer_` in order to get 30 ms. + borrowed_samples_per_channel = + static_cast(required_samples - decoded_length_per_channel); + memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels], + decoded_buffer, sizeof(int16_t) * decoded_length); + sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel, + decoded_buffer); + decoded_length = required_samples * num_channels; + } + + size_t samples_removed = 0; + Accelerate::ReturnCodes return_code = + accelerate_->Process(decoded_buffer, decoded_length, fast_accelerate, + algorithm_buffer_.get(), &samples_removed); + stats_->AcceleratedSamples(samples_removed); + switch (return_code) { + case Accelerate::kSuccess: + last_mode_ = Mode::kAccelerateSuccess; + break; + case Accelerate::kSuccessLowEnergy: + last_mode_ = Mode::kAccelerateLowEnergy; + break; + case Accelerate::kNoStretch: + last_mode_ = Mode::kAccelerateFail; + break; + case Accelerate::kError: + // TODO(hlundin): Map to Modes::kError instead? + last_mode_ = Mode::kAccelerateFail; + return kAccelerateError; + } + + if (borrowed_samples_per_channel > 0) { + // Copy borrowed samples back to the `sync_buffer_`. + size_t length = algorithm_buffer_->Size(); + if (length < borrowed_samples_per_channel) { + // This destroys the beginning of the buffer, but will not cause any + // problems. + sync_buffer_->ReplaceAtIndex( + *algorithm_buffer_, + sync_buffer_->Size() - borrowed_samples_per_channel); + sync_buffer_->PushFrontZeros(borrowed_samples_per_channel - length); + algorithm_buffer_->PopFront(length); + RTC_DCHECK(algorithm_buffer_->Empty()); + } else { + sync_buffer_->ReplaceAtIndex( + *algorithm_buffer_, borrowed_samples_per_channel, + sync_buffer_->Size() - borrowed_samples_per_channel); + algorithm_buffer_->PopFront(borrowed_samples_per_channel); + } + } + + // If last packet was decoded as an inband CNG, set mode to CNG instead. + if (speech_type == AudioDecoder::kComfortNoise) { + last_mode_ = Mode::kCodecInternalCng; + } + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } + expand_->Reset(); + return 0; +} + +int NetEqImpl::DoPreemptiveExpand(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) { + const size_t required_samples = + static_cast(240 * fs_mult_); // Must have 30 ms. + size_t num_channels = algorithm_buffer_->Channels(); + size_t borrowed_samples_per_channel = 0; + size_t old_borrowed_samples_per_channel = 0; + size_t decoded_length_per_channel = decoded_length / num_channels; + if (decoded_length_per_channel < required_samples) { + // Must move data from the `sync_buffer_` in order to get 30 ms. + borrowed_samples_per_channel = + required_samples - decoded_length_per_channel; + // Calculate how many of these were already played out. + old_borrowed_samples_per_channel = + (borrowed_samples_per_channel > sync_buffer_->FutureLength()) + ? (borrowed_samples_per_channel - sync_buffer_->FutureLength()) + : 0; + memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels], + decoded_buffer, sizeof(int16_t) * decoded_length); + sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel, + decoded_buffer); + decoded_length = required_samples * num_channels; + } + + size_t samples_added = 0; + PreemptiveExpand::ReturnCodes return_code = preemptive_expand_->Process( + decoded_buffer, decoded_length, old_borrowed_samples_per_channel, + algorithm_buffer_.get(), &samples_added); + stats_->PreemptiveExpandedSamples(samples_added); + switch (return_code) { + case PreemptiveExpand::kSuccess: + last_mode_ = Mode::kPreemptiveExpandSuccess; + break; + case PreemptiveExpand::kSuccessLowEnergy: + last_mode_ = Mode::kPreemptiveExpandLowEnergy; + break; + case PreemptiveExpand::kNoStretch: + last_mode_ = Mode::kPreemptiveExpandFail; + break; + case PreemptiveExpand::kError: + // TODO(hlundin): Map to Modes::kError instead? + last_mode_ = Mode::kPreemptiveExpandFail; + return kPreemptiveExpandError; + } + + if (borrowed_samples_per_channel > 0) { + // Copy borrowed samples back to the `sync_buffer_`. + sync_buffer_->ReplaceAtIndex( + *algorithm_buffer_, borrowed_samples_per_channel, + sync_buffer_->Size() - borrowed_samples_per_channel); + algorithm_buffer_->PopFront(borrowed_samples_per_channel); + } + + // If last packet was decoded as an inband CNG, set mode to CNG instead. + if (speech_type == AudioDecoder::kComfortNoise) { + last_mode_ = Mode::kCodecInternalCng; + } + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } + expand_->Reset(); + return 0; +} + +int NetEqImpl::DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) { + if (!packet_list->empty()) { + // Must have exactly one SID frame at this point. + RTC_DCHECK_EQ(packet_list->size(), 1); + const Packet& packet = packet_list->front(); + if (!decoder_database_->IsComfortNoise(packet.payload_type)) { + RTC_LOG(LS_ERROR) << "Trying to decode non-CNG payload as CNG."; + return kOtherError; + } + if (comfort_noise_->UpdateParameters(packet) == + ComfortNoise::kInternalError) { + algorithm_buffer_->Zeros(output_size_samples_); + return -comfort_noise_->internal_error_code(); + } + } + int cn_return = + comfort_noise_->Generate(output_size_samples_, algorithm_buffer_.get()); + expand_->Reset(); + last_mode_ = Mode::kRfc3389Cng; + if (!play_dtmf) { + dtmf_tone_generator_->Reset(); + } + if (cn_return == ComfortNoise::kInternalError) { + RTC_LOG(LS_WARNING) << "Comfort noise generator returned error code: " + << comfort_noise_->internal_error_code(); + return kComfortNoiseErrorCode; + } else if (cn_return == ComfortNoise::kUnknownPayloadType) { + return kUnknownRtpPayloadType; + } + return 0; +} + +void NetEqImpl::DoCodecInternalCng(const int16_t* decoded_buffer, + size_t decoded_length) { + RTC_DCHECK(normal_.get()); + normal_->Process(decoded_buffer, decoded_length, last_mode_, + algorithm_buffer_.get()); + last_mode_ = Mode::kCodecInternalCng; + expand_->Reset(); +} + +int NetEqImpl::DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) { + // This block of the code and the block further down, handling `dtmf_switch` + // are commented out. Otherwise playing out-of-band DTMF would fail in VoE + // test, DtmfTest.ManualSuccessfullySendsOutOfBandTelephoneEvents. This is + // equivalent to `dtmf_switch` always be false. + // + // See http://webrtc-codereview.appspot.com/1195004/ for discussion + // On this issue. This change might cause some glitches at the point of + // switch from audio to DTMF. Issue 1545 is filed to track this. + // + // bool dtmf_switch = false; + // if ((last_mode_ != Modes::kDtmf) && + // dtmf_tone_generator_->initialized()) { + // // Special case; see below. + // // We must catch this before calling Generate, since `initialized` is + // // modified in that call. + // dtmf_switch = true; + // } + + int dtmf_return_value = 0; + if (!dtmf_tone_generator_->initialized()) { + // Initialize if not already done. + dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no, + dtmf_event.volume); + } + + if (dtmf_return_value == 0) { + // Generate DTMF signal. + dtmf_return_value = dtmf_tone_generator_->Generate(output_size_samples_, + algorithm_buffer_.get()); + } + + if (dtmf_return_value < 0) { + algorithm_buffer_->Zeros(output_size_samples_); + return dtmf_return_value; + } + + // if (dtmf_switch) { + // // This is the special case where the previous operation was DTMF + // // overdub, but the current instruction is "regular" DTMF. We must make + // // sure that the DTMF does not have any discontinuities. The first DTMF + // // sample that we generate now must be played out immediately, therefore + // // it must be copied to the speech buffer. + // // TODO(hlundin): This code seems incorrect. (Legacy.) Write test and + // // verify correct operation. + // RTC_DCHECK_NOTREACHED(); + // // Must generate enough data to replace all of the `sync_buffer_` + // // "future". + // int required_length = sync_buffer_->FutureLength(); + // RTC_DCHECK(dtmf_tone_generator_->initialized()); + // dtmf_return_value = dtmf_tone_generator_->Generate(required_length, + // algorithm_buffer_); + // RTC_DCHECK((size_t) required_length == algorithm_buffer_->Size()); + // if (dtmf_return_value < 0) { + // algorithm_buffer_->Zeros(output_size_samples_); + // return dtmf_return_value; + // } + // + // // Overwrite the "future" part of the speech buffer with the new DTMF + // // data. + // // TODO(hlundin): It seems that this overwriting has gone lost. + // // Not adapted for multi-channel yet. + // RTC_DCHECK(algorithm_buffer_->Channels() == 1); + // if (algorithm_buffer_->Channels() != 1) { + // RTC_LOG(LS_WARNING) << "DTMF not supported for more than one channel"; + // return kStereoNotSupported; + // } + // // Shuffle the remaining data to the beginning of algorithm buffer. + // algorithm_buffer_->PopFront(sync_buffer_->FutureLength()); + // } + + sync_buffer_->IncreaseEndTimestamp( + static_cast(output_size_samples_)); + expand_->Reset(); + last_mode_ = Mode::kDtmf; + + // Set to false because the DTMF is already in the algorithm buffer. + *play_dtmf = false; + return 0; +} + +int NetEqImpl::DtmfOverdub(const DtmfEvent& dtmf_event, + size_t num_channels, + int16_t* output) const { + size_t out_index = 0; + size_t overdub_length = output_size_samples_; // Default value. + + if (sync_buffer_->dtmf_index() > sync_buffer_->next_index()) { + // Special operation for transition from "DTMF only" to "DTMF overdub". + out_index = + std::min(sync_buffer_->dtmf_index() - sync_buffer_->next_index(), + output_size_samples_); + overdub_length = output_size_samples_ - out_index; + } + + AudioMultiVector dtmf_output(num_channels); + int dtmf_return_value = 0; + if (!dtmf_tone_generator_->initialized()) { + dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no, + dtmf_event.volume); + } + if (dtmf_return_value == 0) { + dtmf_return_value = + dtmf_tone_generator_->Generate(overdub_length, &dtmf_output); + RTC_DCHECK_EQ(overdub_length, dtmf_output.Size()); + } + dtmf_output.ReadInterleaved(overdub_length, &output[out_index]); + return dtmf_return_value < 0 ? dtmf_return_value : 0; +} + +int NetEqImpl::ExtractPackets(size_t required_samples, + PacketList* packet_list) { + bool first_packet = true; + uint8_t prev_payload_type = 0; + uint32_t prev_timestamp = 0; + uint16_t prev_sequence_number = 0; + bool next_packet_available = false; + + const Packet* next_packet = packet_buffer_->PeekNextPacket(); + RTC_DCHECK(next_packet); + if (!next_packet) { + RTC_LOG(LS_ERROR) << "Packet buffer unexpectedly empty."; + return -1; + } + uint32_t first_timestamp = next_packet->timestamp; + size_t extracted_samples = 0; + + // Packet extraction loop. + do { + timestamp_ = next_packet->timestamp; + absl::optional packet = packet_buffer_->GetNextPacket(); + // `next_packet` may be invalid after the `packet_buffer_` operation. + next_packet = nullptr; + if (!packet) { + RTC_LOG(LS_ERROR) << "Should always be able to extract a packet here"; + RTC_DCHECK_NOTREACHED(); // Should always be able to extract a packet + // here. + return -1; + } + const uint64_t waiting_time_ms = packet->waiting_time->ElapsedMs(); + stats_->StoreWaitingTime(waiting_time_ms); + RTC_DCHECK(!packet->empty()); + + if (first_packet) { + first_packet = false; + if (nack_enabled_) { + RTC_DCHECK(nack_); + // TODO(henrik.lundin): Should we update this for all decoded packets? + nack_->UpdateLastDecodedPacket(packet->sequence_number, + packet->timestamp); + } + prev_sequence_number = packet->sequence_number; + prev_timestamp = packet->timestamp; + prev_payload_type = packet->payload_type; + } + + const bool has_cng_packet = + decoder_database_->IsComfortNoise(packet->payload_type); + // Store number of extracted samples. + size_t packet_duration = 0; + if (packet->frame) { + packet_duration = packet->frame->Duration(); + // TODO(ossu): Is this the correct way to track Opus FEC packets? + if (packet->priority.codec_level > 0) { + stats_->SecondaryDecodedSamples( + rtc::dchecked_cast(packet_duration)); + } + } else if (!has_cng_packet) { + RTC_LOG(LS_WARNING) << "Unknown payload type " + << static_cast(packet->payload_type); + RTC_DCHECK_NOTREACHED(); + } + + if (packet_duration == 0) { + // Decoder did not return a packet duration. Assume that the packet + // contains the same number of samples as the previous one. + packet_duration = decoder_frame_length_; + } + extracted_samples = packet->timestamp - first_timestamp + packet_duration; + + RTC_DCHECK(controller_); + stats_->JitterBufferDelay(packet_duration, waiting_time_ms, + controller_->TargetLevelMs(), + controller_->UnlimitedTargetLevelMs()); + + packet_list->push_back(std::move(*packet)); // Store packet in list. + packet = absl::nullopt; // Ensure it's never used after the move. + + // Check what packet is available next. + next_packet = packet_buffer_->PeekNextPacket(); + next_packet_available = false; + if (next_packet && prev_payload_type == next_packet->payload_type && + !has_cng_packet) { + int16_t seq_no_diff = next_packet->sequence_number - prev_sequence_number; + size_t ts_diff = next_packet->timestamp - prev_timestamp; + if ((seq_no_diff == 1 || seq_no_diff == 0) && + ts_diff <= packet_duration) { + // The next sequence number is available, or the next part of a packet + // that was split into pieces upon insertion. + next_packet_available = true; + } + prev_sequence_number = next_packet->sequence_number; + prev_timestamp = next_packet->timestamp; + } + } while (extracted_samples < required_samples && next_packet_available); + + if (extracted_samples > 0) { + // Delete old packets only when we are going to decode something. Otherwise, + // we could end up in the situation where we never decode anything, since + // all incoming packets are considered too old but the buffer will also + // never be flooded and flushed. + packet_buffer_->DiscardAllOldPackets(timestamp_, stats_.get()); + } + + return rtc::dchecked_cast(extracted_samples); +} + +void NetEqImpl::UpdatePlcComponents(int fs_hz, size_t channels) { + // Delete objects and create new ones. + expand_.reset(expand_factory_->Create(background_noise_.get(), + sync_buffer_.get(), &random_vector_, + stats_.get(), fs_hz, channels)); + merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get())); +} + +void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) { + RTC_LOG(LS_VERBOSE) << "SetSampleRateAndChannels " << fs_hz << " " + << channels; + // TODO(hlundin): Change to an enumerator and skip assert. + RTC_DCHECK(fs_hz == 8000 || fs_hz == 16000 || fs_hz == 32000 || + fs_hz == 48000); + RTC_DCHECK_GT(channels, 0); + + // Before changing the sample rate, end and report any ongoing expand event. + stats_->EndExpandEvent(fs_hz_); + fs_hz_ = fs_hz; + fs_mult_ = fs_hz / 8000; + output_size_samples_ = static_cast(kOutputSizeMs * 8 * fs_mult_); + decoder_frame_length_ = 3 * output_size_samples_; // Initialize to 30ms. + + last_mode_ = Mode::kNormal; + + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + if (cng_decoder) + cng_decoder->Reset(); + + // Reinit post-decode VAD with new sample rate. + RTC_DCHECK(vad_.get()); // Cannot be NULL here. + vad_->Init(); + + // Delete algorithm buffer and create a new one. + algorithm_buffer_.reset(new AudioMultiVector(channels)); + + // Delete sync buffer and create a new one. + sync_buffer_.reset(new SyncBuffer(channels, kSyncBufferSize * fs_mult_)); + + // Delete BackgroundNoise object and create a new one. + background_noise_.reset(new BackgroundNoise(channels)); + + // Reset random vector. + random_vector_.Reset(); + + UpdatePlcComponents(fs_hz, channels); + + // Move index so that we create a small set of future samples (all 0). + sync_buffer_->set_next_index(sync_buffer_->next_index() - + expand_->overlap_length()); + + normal_.reset(new Normal(fs_hz, decoder_database_.get(), *background_noise_, + expand_.get(), stats_.get())); + accelerate_.reset( + accelerate_factory_->Create(fs_hz, channels, *background_noise_)); + preemptive_expand_.reset(preemptive_expand_factory_->Create( + fs_hz, channels, *background_noise_, expand_->overlap_length())); + + // Delete ComfortNoise object and create a new one. + comfort_noise_.reset( + new ComfortNoise(fs_hz, decoder_database_.get(), sync_buffer_.get())); + + // Verify that `decoded_buffer_` is long enough. + if (decoded_buffer_length_ < kMaxFrameSize * channels) { + // Reallocate to larger size. + decoded_buffer_length_ = kMaxFrameSize * channels; + decoded_buffer_.reset(new int16_t[decoded_buffer_length_]); + } + RTC_CHECK(controller_) << "Unexpectedly found no NetEqController"; + controller_->SetSampleRate(fs_hz_, output_size_samples_); +} + +NetEqImpl::OutputType NetEqImpl::LastOutputType() { + RTC_DCHECK(vad_.get()); + RTC_DCHECK(expand_.get()); + if (last_mode_ == Mode::kCodecInternalCng || + last_mode_ == Mode::kRfc3389Cng) { + return OutputType::kCNG; + } else if (last_mode_ == Mode::kExpand && expand_->MuteFactor(0) == 0) { + // Expand mode has faded down to background noise only (very long expand). + return OutputType::kPLCCNG; + } else if (last_mode_ == Mode::kExpand) { + return OutputType::kPLC; + } else if (vad_->running() && !vad_->active_speech()) { + return OutputType::kVadPassive; + } else if (last_mode_ == Mode::kCodecPlc) { + return OutputType::kCodecPLC; + } else { + return OutputType::kNormalSpeech; + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.h b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.h new file mode 100644 index 0000000000..6120eab5b6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.h @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ +#define MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ + +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/audio/audio_frame.h" +#include "api/neteq/neteq.h" +#include "api/neteq/neteq_controller.h" +#include "api/neteq/neteq_controller_factory.h" +#include "api/neteq/tick_timer.h" +#include "api/rtp_packet_info.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/expand_uma_logger.h" +#include "modules/audio_coding/neteq/packet.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +// Forward declarations. +class Accelerate; +class BackgroundNoise; +class Clock; +class ComfortNoise; +class DecoderDatabase; +class DtmfBuffer; +class DtmfToneGenerator; +class Expand; +class Merge; +class NackTracker; +class Normal; +class PacketBuffer; +class RedPayloadSplitter; +class PostDecodeVad; +class PreemptiveExpand; +class RandomVector; +class SyncBuffer; +class TimestampScaler; +struct AccelerateFactory; +struct DtmfEvent; +struct ExpandFactory; +struct PreemptiveExpandFactory; + +class NetEqImpl : public webrtc::NetEq { + public: + enum class OutputType { + kNormalSpeech, + kPLC, + kCNG, + kPLCCNG, + kVadPassive, + kCodecPLC + }; + + enum ErrorCodes { + kNoError = 0, + kOtherError, + kUnknownRtpPayloadType, + kDecoderNotFound, + kInvalidPointer, + kAccelerateError, + kPreemptiveExpandError, + kComfortNoiseErrorCode, + kDecoderErrorCode, + kOtherDecoderError, + kInvalidOperation, + kDtmfParsingError, + kDtmfInsertError, + kSampleUnderrun, + kDecodedTooMuch, + kRedundancySplitError, + kPacketBufferCorruption + }; + + struct Dependencies { + // The constructor populates the Dependencies struct with the default + // implementations of the objects. They can all be replaced by the user + // before sending the struct to the NetEqImpl constructor. However, there + // are dependencies between some of the classes inside the struct, so + // swapping out one may make it necessary to re-create another one. + Dependencies(const NetEq::Config& config, + Clock* clock, + const rtc::scoped_refptr& decoder_factory, + const NetEqControllerFactory& controller_factory); + ~Dependencies(); + + Clock* const clock; + std::unique_ptr tick_timer; + std::unique_ptr stats; + std::unique_ptr decoder_database; + std::unique_ptr dtmf_buffer; + std::unique_ptr dtmf_tone_generator; + std::unique_ptr packet_buffer; + std::unique_ptr neteq_controller; + std::unique_ptr red_payload_splitter; + std::unique_ptr timestamp_scaler; + std::unique_ptr accelerate_factory; + std::unique_ptr expand_factory; + std::unique_ptr preemptive_expand_factory; + }; + + // Creates a new NetEqImpl object. + NetEqImpl(const NetEq::Config& config, + Dependencies&& deps, + bool create_components = true); + + ~NetEqImpl() override; + + NetEqImpl(const NetEqImpl&) = delete; + NetEqImpl& operator=(const NetEqImpl&) = delete; + + // Inserts a new packet into NetEq. Returns 0 on success, -1 on failure. + int InsertPacket(const RTPHeader& rtp_header, + rtc::ArrayView payload) override; + + void InsertEmptyPacket(const RTPHeader& rtp_header) override; + + int GetAudio( + AudioFrame* audio_frame, + bool* muted, + int* current_sample_rate_hz = nullptr, + absl::optional action_override = absl::nullopt) override; + + void SetCodecs(const std::map& codecs) override; + + bool RegisterPayloadType(int rtp_payload_type, + const SdpAudioFormat& audio_format) override; + + // Removes `rtp_payload_type` from the codec database. Returns 0 on success, + // -1 on failure. + int RemovePayloadType(uint8_t rtp_payload_type) override; + + void RemoveAllPayloadTypes() override; + + bool SetMinimumDelay(int delay_ms) override; + + bool SetMaximumDelay(int delay_ms) override; + + bool SetBaseMinimumDelayMs(int delay_ms) override; + + int GetBaseMinimumDelayMs() const override; + + int TargetDelayMs() const override; + + int FilteredCurrentDelayMs() const override; + + // Writes the current network statistics to `stats`. The statistics are reset + // after the call. + int NetworkStatistics(NetEqNetworkStatistics* stats) override; + + NetEqNetworkStatistics CurrentNetworkStatistics() const override; + + NetEqLifetimeStatistics GetLifetimeStatistics() const override; + + NetEqOperationsAndState GetOperationsAndState() const override; + + // Enables post-decode VAD. When enabled, GetAudio() will return + // kOutputVADPassive when the signal contains no speech. + void EnableVad() override; + + // Disables post-decode VAD. + void DisableVad() override; + + absl::optional GetPlayoutTimestamp() const override; + + int last_output_sample_rate_hz() const override; + + absl::optional GetDecoderFormat( + int payload_type) const override; + + // Flushes both the packet buffer and the sync buffer. + void FlushBuffers() override; + + void EnableNack(size_t max_nack_list_size) override; + + void DisableNack() override; + + std::vector GetNackList(int64_t round_trip_time_ms) const override; + + int SyncBufferSizeMs() const override; + + // This accessor method is only intended for testing purposes. + const SyncBuffer* sync_buffer_for_test() const; + Operation last_operation_for_test() const; + + protected: + static const int kOutputSizeMs = 10; + static const size_t kMaxFrameSize = 5760; // 120 ms @ 48 kHz. + // TODO(hlundin): Provide a better value for kSyncBufferSize. + // Current value is kMaxFrameSize + 60 ms * 48 kHz, which is enough for + // calculating correlations of current frame against history. + static const size_t kSyncBufferSize = kMaxFrameSize + 60 * 48; + + // Inserts a new packet into NetEq. This is used by the InsertPacket method + // above. Returns 0 on success, otherwise an error code. + // TODO(hlundin): Merge this with InsertPacket above? + int InsertPacketInternal(const RTPHeader& rtp_header, + rtc::ArrayView payload) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Delivers 10 ms of audio data. The data is written to `audio_frame`. + // Returns 0 on success, otherwise an error code. + int GetAudioInternal(AudioFrame* audio_frame, + bool* muted, + absl::optional action_override) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Provides a decision to the GetAudioInternal method. The decision what to + // do is written to `operation`. Packets to decode are written to + // `packet_list`, and a DTMF event to play is written to `dtmf_event`. When + // DTMF should be played, `play_dtmf` is set to true by the method. + // Returns 0 on success, otherwise an error code. + int GetDecision(Operation* operation, + PacketList* packet_list, + DtmfEvent* dtmf_event, + bool* play_dtmf, + absl::optional action_override) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Decodes the speech packets in `packet_list`, and writes the results to + // `decoded_buffer`, which is allocated to hold `decoded_buffer_length` + // elements. The length of the decoded data is written to `decoded_length`. + // The speech type -- speech or (codec-internal) comfort noise -- is written + // to `speech_type`. If `packet_list` contains any SID frames for RFC 3389 + // comfort noise, those are not decoded. + int Decode(PacketList* packet_list, + Operation* operation, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Sub-method to Decode(). Performs codec internal CNG. + int DecodeCng(AudioDecoder* decoder, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Sub-method to Decode(). Performs the actual decoding. + int DecodeLoop(PacketList* packet_list, + const Operation& operation, + AudioDecoder* decoder, + int* decoded_length, + AudioDecoder::SpeechType* speech_type) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Sub-method which calls the Normal class to perform the normal operation. + void DoNormal(const int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Sub-method which calls the Merge class to perform the merge operation. + void DoMerge(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + bool DoCodecPlc() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Sub-method which calls the Expand class to perform the expand operation. + int DoExpand(bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Sub-method which calls the Accelerate class to perform the accelerate + // operation. + int DoAccelerate(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf, + bool fast_accelerate) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Sub-method which calls the PreemptiveExpand class to perform the + // preemtive expand operation. + int DoPreemptiveExpand(int16_t* decoded_buffer, + size_t decoded_length, + AudioDecoder::SpeechType speech_type, + bool play_dtmf) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort + // noise. `packet_list` can either contain one SID frame to update the + // noise parameters, or no payload at all, in which case the previously + // received parameters are used. + int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Calls the audio decoder to generate codec-internal comfort noise when + // no packet was received. + void DoCodecInternalCng(const int16_t* decoded_buffer, size_t decoded_length) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Calls the DtmfToneGenerator class to generate DTMF tones. + int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Overdub DTMF on top of `output`. + int DtmfOverdub(const DtmfEvent& dtmf_event, + size_t num_channels, + int16_t* output) const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Extracts packets from `packet_buffer_` to produce at least + // `required_samples` samples. The packets are inserted into `packet_list`. + // Returns the number of samples that the packets in the list will produce, or + // -1 in case of an error. + int ExtractPackets(size_t required_samples, PacketList* packet_list) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Resets various variables and objects to new values based on the sample rate + // `fs_hz` and `channels` number audio channels. + void SetSampleRateAndChannels(int fs_hz, size_t channels) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Returns the output type for the audio produced by the latest call to + // GetAudio(). + OutputType LastOutputType() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Updates Expand and Merge. + virtual void UpdatePlcComponents(int fs_hz, size_t channels) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + NetEqNetworkStatistics CurrentNetworkStatisticsInternal() const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + Clock* const clock_; + + mutable Mutex mutex_; + const std::unique_ptr tick_timer_ RTC_GUARDED_BY(mutex_); + const std::unique_ptr decoder_database_ + RTC_GUARDED_BY(mutex_); + const std::unique_ptr dtmf_buffer_ RTC_GUARDED_BY(mutex_); + const std::unique_ptr dtmf_tone_generator_ + RTC_GUARDED_BY(mutex_); + const std::unique_ptr packet_buffer_ RTC_GUARDED_BY(mutex_); + const std::unique_ptr red_payload_splitter_ + RTC_GUARDED_BY(mutex_); + const std::unique_ptr timestamp_scaler_ + RTC_GUARDED_BY(mutex_); + const std::unique_ptr vad_ RTC_GUARDED_BY(mutex_); + const std::unique_ptr expand_factory_ RTC_GUARDED_BY(mutex_); + const std::unique_ptr accelerate_factory_ + RTC_GUARDED_BY(mutex_); + const std::unique_ptr preemptive_expand_factory_ + RTC_GUARDED_BY(mutex_); + const std::unique_ptr stats_ RTC_GUARDED_BY(mutex_); + + std::unique_ptr background_noise_ RTC_GUARDED_BY(mutex_); + std::unique_ptr controller_ RTC_GUARDED_BY(mutex_); + std::unique_ptr algorithm_buffer_ RTC_GUARDED_BY(mutex_); + std::unique_ptr sync_buffer_ RTC_GUARDED_BY(mutex_); + std::unique_ptr expand_ RTC_GUARDED_BY(mutex_); + std::unique_ptr normal_ RTC_GUARDED_BY(mutex_); + std::unique_ptr merge_ RTC_GUARDED_BY(mutex_); + std::unique_ptr accelerate_ RTC_GUARDED_BY(mutex_); + std::unique_ptr preemptive_expand_ RTC_GUARDED_BY(mutex_); + RandomVector random_vector_ RTC_GUARDED_BY(mutex_); + std::unique_ptr comfort_noise_ RTC_GUARDED_BY(mutex_); + int fs_hz_ RTC_GUARDED_BY(mutex_); + int fs_mult_ RTC_GUARDED_BY(mutex_); + int last_output_sample_rate_hz_ RTC_GUARDED_BY(mutex_); + size_t output_size_samples_ RTC_GUARDED_BY(mutex_); + size_t decoder_frame_length_ RTC_GUARDED_BY(mutex_); + Mode last_mode_ RTC_GUARDED_BY(mutex_); + Operation last_operation_ RTC_GUARDED_BY(mutex_); + size_t decoded_buffer_length_ RTC_GUARDED_BY(mutex_); + std::unique_ptr decoded_buffer_ RTC_GUARDED_BY(mutex_); + uint32_t playout_timestamp_ RTC_GUARDED_BY(mutex_); + bool new_codec_ RTC_GUARDED_BY(mutex_); + uint32_t timestamp_ RTC_GUARDED_BY(mutex_); + bool reset_decoder_ RTC_GUARDED_BY(mutex_); + absl::optional current_rtp_payload_type_ RTC_GUARDED_BY(mutex_); + absl::optional current_cng_rtp_payload_type_ RTC_GUARDED_BY(mutex_); + bool first_packet_ RTC_GUARDED_BY(mutex_); + bool enable_fast_accelerate_ RTC_GUARDED_BY(mutex_); + std::unique_ptr nack_ RTC_GUARDED_BY(mutex_); + bool nack_enabled_ RTC_GUARDED_BY(mutex_); + const bool enable_muted_state_ RTC_GUARDED_BY(mutex_); + AudioFrame::VADActivity last_vad_activity_ RTC_GUARDED_BY(mutex_) = + AudioFrame::kVadPassive; + std::unique_ptr generated_noise_stopwatch_ + RTC_GUARDED_BY(mutex_); + std::vector last_decoded_packet_infos_ RTC_GUARDED_BY(mutex_); + ExpandUmaLogger expand_uma_logger_ RTC_GUARDED_BY(mutex_); + ExpandUmaLogger speech_expand_uma_logger_ RTC_GUARDED_BY(mutex_); + bool no_time_stretching_ RTC_GUARDED_BY(mutex_); // Only used for test. + rtc::BufferT concealment_audio_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc new file mode 100644 index 0000000000..ce2be656ef --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl_unittest.cc @@ -0,0 +1,1871 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/neteq_impl.h" + +#include +#include +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/neteq/default_neteq_controller_factory.h" +#include "api/neteq/neteq.h" +#include "api/neteq/neteq_controller.h" +#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h" +#include "modules/audio_coding/neteq/accelerate.h" +#include "modules/audio_coding/neteq/decision_logic.h" +#include "modules/audio_coding/neteq/default_neteq_factory.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/histogram.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/mock/mock_dtmf_buffer.h" +#include "modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h" +#include "modules/audio_coding/neteq/mock/mock_neteq_controller.h" +#include "modules/audio_coding/neteq/mock/mock_packet_buffer.h" +#include "modules/audio_coding/neteq/mock/mock_red_payload_splitter.h" +#include "modules/audio_coding/neteq/preemptive_expand.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "modules/audio_coding/neteq/timestamp_scaler.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/clock.h" +#include "test/audio_decoder_proxy_factory.h" +#include "test/function_audio_decoder_factory.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder.h" +#include "test/mock_audio_decoder_factory.h" + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::DoAll; +using ::testing::ElementsAre; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::IsEmpty; +using ::testing::IsNull; +using ::testing::Pointee; +using ::testing::Return; +using ::testing::ReturnNull; +using ::testing::SetArgPointee; +using ::testing::SetArrayArgument; +using ::testing::SizeIs; +using ::testing::WithArg; + +namespace webrtc { + +// This function is called when inserting a packet list into the mock packet +// buffer. The purpose is to delete all inserted packets properly, to avoid +// memory leaks in the test. +int DeletePacketsAndReturnOk(PacketList* packet_list) { + packet_list->clear(); + return PacketBuffer::kOK; +} + +class NetEqImplTest : public ::testing::Test { + protected: + NetEqImplTest() : clock_(0) { config_.sample_rate_hz = 8000; } + + void CreateInstance( + const rtc::scoped_refptr& decoder_factory) { + ASSERT_TRUE(decoder_factory); + config_.enable_muted_state = enable_muted_state_; + NetEqImpl::Dependencies deps(config_, &clock_, decoder_factory, + DefaultNetEqControllerFactory()); + + // Get a local pointer to NetEq's TickTimer object. + tick_timer_ = deps.tick_timer.get(); + + if (use_mock_decoder_database_) { + std::unique_ptr mock(new MockDecoderDatabase); + mock_decoder_database_ = mock.get(); + EXPECT_CALL(*mock_decoder_database_, GetActiveCngDecoder()) + .WillOnce(ReturnNull()); + deps.decoder_database = std::move(mock); + } + decoder_database_ = deps.decoder_database.get(); + + if (use_mock_dtmf_buffer_) { + std::unique_ptr mock( + new MockDtmfBuffer(config_.sample_rate_hz)); + mock_dtmf_buffer_ = mock.get(); + deps.dtmf_buffer = std::move(mock); + } + dtmf_buffer_ = deps.dtmf_buffer.get(); + + if (use_mock_dtmf_tone_generator_) { + std::unique_ptr mock(new MockDtmfToneGenerator); + mock_dtmf_tone_generator_ = mock.get(); + deps.dtmf_tone_generator = std::move(mock); + } + dtmf_tone_generator_ = deps.dtmf_tone_generator.get(); + + if (use_mock_packet_buffer_) { + std::unique_ptr mock( + new MockPacketBuffer(config_.max_packets_in_buffer, tick_timer_)); + mock_packet_buffer_ = mock.get(); + deps.packet_buffer = std::move(mock); + } + packet_buffer_ = deps.packet_buffer.get(); + + if (use_mock_neteq_controller_) { + std::unique_ptr mock(new MockNetEqController()); + mock_neteq_controller_ = mock.get(); + deps.neteq_controller = std::move(mock); + } else { + deps.stats = std::make_unique(); + NetEqController::Config controller_config; + controller_config.tick_timer = tick_timer_; + controller_config.base_min_delay_ms = config_.min_delay_ms; + controller_config.allow_time_stretching = true; + controller_config.max_packets_in_buffer = config_.max_packets_in_buffer; + controller_config.clock = &clock_; + deps.neteq_controller = + std::make_unique(std::move(controller_config)); + } + neteq_controller_ = deps.neteq_controller.get(); + + if (use_mock_payload_splitter_) { + std::unique_ptr mock(new MockRedPayloadSplitter); + mock_payload_splitter_ = mock.get(); + deps.red_payload_splitter = std::move(mock); + } + red_payload_splitter_ = deps.red_payload_splitter.get(); + + deps.timestamp_scaler = std::unique_ptr( + new TimestampScaler(*deps.decoder_database.get())); + + neteq_.reset(new NetEqImpl(config_, std::move(deps))); + ASSERT_TRUE(neteq_ != NULL); + } + + void CreateInstance() { CreateInstance(CreateBuiltinAudioDecoderFactory()); } + + void UseNoMocks() { + ASSERT_TRUE(neteq_ == NULL) << "Must call UseNoMocks before CreateInstance"; + use_mock_decoder_database_ = false; + use_mock_neteq_controller_ = false; + use_mock_dtmf_buffer_ = false; + use_mock_dtmf_tone_generator_ = false; + use_mock_packet_buffer_ = false; + use_mock_payload_splitter_ = false; + } + + virtual ~NetEqImplTest() { + if (use_mock_decoder_database_) { + EXPECT_CALL(*mock_decoder_database_, Die()).Times(1); + } + if (use_mock_neteq_controller_) { + EXPECT_CALL(*mock_neteq_controller_, Die()).Times(1); + } + if (use_mock_dtmf_buffer_) { + EXPECT_CALL(*mock_dtmf_buffer_, Die()).Times(1); + } + if (use_mock_dtmf_tone_generator_) { + EXPECT_CALL(*mock_dtmf_tone_generator_, Die()).Times(1); + } + if (use_mock_packet_buffer_) { + EXPECT_CALL(*mock_packet_buffer_, Die()).Times(1); + } + } + + void TestDtmfPacket(int sample_rate_hz) { + const size_t kPayloadLength = 4; + const uint8_t kPayloadType = 110; + const int kSampleRateHz = 16000; + config_.sample_rate_hz = kSampleRateHz; + UseNoMocks(); + CreateInstance(); + // Event: 2, E bit, Volume: 17, Length: 4336. + uint8_t payload[kPayloadLength] = {0x02, 0x80 + 0x11, 0x10, 0xF0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_TRUE(neteq_->RegisterPayloadType( + kPayloadType, SdpAudioFormat("telephone-event", sample_rate_hz, 1))); + + // Insert first packet. + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + + // Pull audio once. + const size_t kMaxOutputSize = + static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_FALSE(muted); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // DTMF packets are immediately consumed by `InsertPacket()` and won't be + // returned by `GetAudio()`. + EXPECT_THAT(output.packet_infos_, IsEmpty()); + + // Verify first 64 samples of actual output. + const std::vector kOutput( + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -1578, -2816, -3460, -3403, -2709, -1594, -363, 671, 1269, 1328, + 908, 202, -513, -964, -955, -431, 504, 1617, 2602, 3164, + 3101, 2364, 1073, -511, -2047, -3198, -3721, -3525, -2688, -1440, + -99, 1015, 1663, 1744, 1319, 588, -171, -680, -747, -315, + 515, 1512, 2378, 2828, 2674, 1877, 568, -986, -2446, -3482, + -3864, -3516, -2534, -1163}); + ASSERT_GE(kMaxOutputSize, kOutput.size()); + EXPECT_TRUE(std::equal(kOutput.begin(), kOutput.end(), output.data())); + } + + std::unique_ptr neteq_; + NetEq::Config config_; + SimulatedClock clock_; + TickTimer* tick_timer_ = nullptr; + MockDecoderDatabase* mock_decoder_database_ = nullptr; + DecoderDatabase* decoder_database_ = nullptr; + bool use_mock_decoder_database_ = true; + MockNetEqController* mock_neteq_controller_ = nullptr; + NetEqController* neteq_controller_ = nullptr; + bool use_mock_neteq_controller_ = true; + MockDtmfBuffer* mock_dtmf_buffer_ = nullptr; + DtmfBuffer* dtmf_buffer_ = nullptr; + bool use_mock_dtmf_buffer_ = true; + MockDtmfToneGenerator* mock_dtmf_tone_generator_ = nullptr; + DtmfToneGenerator* dtmf_tone_generator_ = nullptr; + bool use_mock_dtmf_tone_generator_ = true; + MockPacketBuffer* mock_packet_buffer_ = nullptr; + PacketBuffer* packet_buffer_ = nullptr; + bool use_mock_packet_buffer_ = true; + MockRedPayloadSplitter* mock_payload_splitter_ = nullptr; + RedPayloadSplitter* red_payload_splitter_ = nullptr; + bool use_mock_payload_splitter_ = true; + bool enable_muted_state_ = false; +}; + +// This tests the interface class NetEq. +// TODO(hlundin): Move to separate file? +TEST(NetEq, CreateAndDestroy) { + NetEq::Config config; + SimulatedClock clock(0); + auto decoder_factory = CreateBuiltinAudioDecoderFactory(); + std::unique_ptr neteq = + DefaultNetEqFactory().CreateNetEq(config, decoder_factory, &clock); +} + +TEST_F(NetEqImplTest, RegisterPayloadType) { + CreateInstance(); + constexpr int rtp_payload_type = 0; + const SdpAudioFormat format("pcmu", 8000, 1); + EXPECT_CALL(*mock_decoder_database_, + RegisterPayload(rtp_payload_type, format)); + neteq_->RegisterPayloadType(rtp_payload_type, format); +} + +TEST_F(NetEqImplTest, RemovePayloadType) { + CreateInstance(); + uint8_t rtp_payload_type = 0; + EXPECT_CALL(*mock_decoder_database_, Remove(rtp_payload_type)) + .WillOnce(Return(DecoderDatabase::kDecoderNotFound)); + // Check that kOK is returned when database returns kDecoderNotFound, because + // removing a payload type that was never registered is not an error. + EXPECT_EQ(NetEq::kOK, neteq_->RemovePayloadType(rtp_payload_type)); +} + +TEST_F(NetEqImplTest, RemoveAllPayloadTypes) { + CreateInstance(); + EXPECT_CALL(*mock_decoder_database_, RemoveAll()).WillOnce(Return()); + neteq_->RemoveAllPayloadTypes(); +} + +TEST_F(NetEqImplTest, InsertPacket) { + using ::testing::AllOf; + using ::testing::Field; + CreateInstance(); + const size_t kPayloadLength = 100; + const uint8_t kPayloadType = 0; + const uint16_t kFirstSequenceNumber = 0x1234; + const uint32_t kFirstTimestamp = 0x12345678; + const uint32_t kSsrc = 0x87654321; + uint8_t payload[kPayloadLength] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = kFirstSequenceNumber; + rtp_header.timestamp = kFirstTimestamp; + rtp_header.ssrc = kSsrc; + Packet fake_packet; + fake_packet.payload_type = kPayloadType; + fake_packet.sequence_number = kFirstSequenceNumber; + fake_packet.timestamp = kFirstTimestamp; + + auto mock_decoder_factory = rtc::make_ref_counted(); + EXPECT_CALL(*mock_decoder_factory, MakeAudioDecoderMock(_, _, _)) + .WillOnce(Invoke([&](const SdpAudioFormat& format, + absl::optional codec_pair_id, + std::unique_ptr* dec) { + EXPECT_EQ("pcmu", format.name); + + std::unique_ptr mock_decoder(new MockAudioDecoder); + EXPECT_CALL(*mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(*mock_decoder, SampleRateHz()).WillRepeatedly(Return(8000)); + EXPECT_CALL(*mock_decoder, Die()).Times(1); // Called when deleted. + + *dec = std::move(mock_decoder); + })); + DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1), + absl::nullopt, mock_decoder_factory.get()); + + // Expectations for decoder database. + EXPECT_CALL(*mock_decoder_database_, GetDecoderInfo(kPayloadType)) + .WillRepeatedly(Return(&info)); + + // Expectations for packet buffer. + EXPECT_CALL(*mock_packet_buffer_, Empty()) + .WillOnce(Return(false)); // Called once after first packet is inserted. + EXPECT_CALL(*mock_packet_buffer_, Flush(_)).Times(1); + EXPECT_CALL(*mock_packet_buffer_, InsertPacketList(_, _, _, _, _, _, _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<2>(kPayloadType), + WithArg<0>(Invoke(DeletePacketsAndReturnOk)))); + // SetArgPointee<2>(kPayloadType) means that the third argument (zero-based + // index) is a pointer, and the variable pointed to is set to kPayloadType. + // Also invoke the function DeletePacketsAndReturnOk to properly delete all + // packets in the list (to avoid memory leaks in the test). + EXPECT_CALL(*mock_packet_buffer_, PeekNextPacket()) + .Times(1) + .WillOnce(Return(&fake_packet)); + + // Expectations for DTMF buffer. + EXPECT_CALL(*mock_dtmf_buffer_, Flush()).Times(1); + + // Expectations for delay manager. + { + // All expectations within this block must be called in this specific order. + InSequence sequence; // Dummy variable. + // Expectations when the first packet is inserted. + EXPECT_CALL( + *mock_neteq_controller_, + PacketArrived( + /*fs_hz*/ 8000, + /*should_update_stats*/ _, + /*info*/ + AllOf( + Field(&NetEqController::PacketArrivedInfo::is_cng_or_dtmf, + false), + Field(&NetEqController::PacketArrivedInfo::main_sequence_number, + kFirstSequenceNumber), + Field(&NetEqController::PacketArrivedInfo::main_timestamp, + kFirstTimestamp)))); + EXPECT_CALL( + *mock_neteq_controller_, + PacketArrived( + /*fs_hz*/ 8000, + /*should_update_stats*/ _, + /*info*/ + AllOf( + Field(&NetEqController::PacketArrivedInfo::is_cng_or_dtmf, + false), + Field(&NetEqController::PacketArrivedInfo::main_sequence_number, + kFirstSequenceNumber + 1), + Field(&NetEqController::PacketArrivedInfo::main_timestamp, + kFirstTimestamp + 160)))); + } + + // Insert first packet. + neteq_->InsertPacket(rtp_header, payload); + + // Insert second packet. + rtp_header.timestamp += 160; + rtp_header.sequenceNumber += 1; + neteq_->InsertPacket(rtp_header, payload); +} + +TEST_F(NetEqImplTest, InsertPacketsUntilBufferIsFull) { + UseNoMocks(); + CreateInstance(); + + const int kPayloadLengthSamples = 80; + const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. + const uint8_t kPayloadType = 17; // Just an arbitrary number. + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("l16", 8000, 1))); + + // Insert packets. The buffer should not flush. + for (size_t i = 1; i <= config_.max_packets_in_buffer; ++i) { + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + rtp_header.timestamp += kPayloadLengthSamples; + rtp_header.sequenceNumber += 1; + EXPECT_EQ(i, packet_buffer_->NumPacketsInBuffer()); + } + + // Insert one more packet and make sure the buffer got flushed. That is, it + // should only hold one single packet. + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + EXPECT_EQ(1u, packet_buffer_->NumPacketsInBuffer()); + const Packet* test_packet = packet_buffer_->PeekNextPacket(); + EXPECT_EQ(rtp_header.timestamp, test_packet->timestamp); + EXPECT_EQ(rtp_header.sequenceNumber, test_packet->sequence_number); +} + +TEST_F(NetEqImplTest, TestDtmfPacketAVT) { + TestDtmfPacket(8000); +} + +TEST_F(NetEqImplTest, TestDtmfPacketAVT16kHz) { + TestDtmfPacket(16000); +} + +TEST_F(NetEqImplTest, TestDtmfPacketAVT32kHz) { + TestDtmfPacket(32000); +} + +TEST_F(NetEqImplTest, TestDtmfPacketAVT48kHz) { + TestDtmfPacket(48000); +} + +// This test verifies that timestamps propagate from the incoming packets +// through to the sync buffer and to the playout timestamp. +TEST_F(NetEqImplTest, VerifyTimestampPropagation) { + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kSampleRateHz = 8000; + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + rtp_header.numCSRCs = 3; + rtp_header.arrOfCSRCs[0] = 43; + rtp_header.arrOfCSRCs[1] = 65; + rtp_header.arrOfCSRCs[2] = 17; + + // This is a dummy decoder that produces as many output samples as the input + // has bytes. The output is an increasing series, starting at 1 for the first + // sample, and then increasing by 1 for each sample. + class CountingSamplesDecoder : public AudioDecoder { + public: + CountingSamplesDecoder() : next_value_(1) {} + + // Produce as many samples as input bytes (`encoded_len`). + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int /* sample_rate_hz */, + int16_t* decoded, + SpeechType* speech_type) override { + for (size_t i = 0; i < encoded_len; ++i) { + decoded[i] = next_value_++; + } + *speech_type = kSpeech; + return rtc::checked_cast(encoded_len); + } + + void Reset() override { next_value_ = 1; } + + int SampleRateHz() const override { return kSampleRateHz; } + + size_t Channels() const override { return 1; } + + uint16_t next_value() const { return next_value_; } + + private: + int16_t next_value_; + } decoder_; + + auto decoder_factory = + rtc::make_ref_counted(&decoder_); + + UseNoMocks(); + CreateInstance(decoder_factory); + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("L16", 8000, 1))); + + // Insert one packet. + clock_.AdvanceTimeMilliseconds(123456); + Timestamp expected_receive_time = clock_.CurrentTime(); + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + + // Pull audio once. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_FALSE(muted); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Verify `output.packet_infos_`. + ASSERT_THAT(output.packet_infos_, SizeIs(1)); + { + const auto& packet_info = output.packet_infos_[0]; + EXPECT_EQ(packet_info.ssrc(), rtp_header.ssrc); + EXPECT_THAT(packet_info.csrcs(), ElementsAre(43, 65, 17)); + EXPECT_EQ(packet_info.rtp_timestamp(), rtp_header.timestamp); + EXPECT_FALSE(packet_info.audio_level().has_value()); + EXPECT_EQ(packet_info.receive_time(), expected_receive_time); + } + + // Start with a simple check that the fake decoder is behaving as expected. + EXPECT_EQ(kPayloadLengthSamples, + static_cast(decoder_.next_value() - 1)); + + // The value of the last of the output samples is the same as the number of + // samples played from the decoded packet. Thus, this number + the RTP + // timestamp should match the playout timestamp. + // Wrap the expected value in an absl::optional to compare them as such. + EXPECT_EQ( + absl::optional(rtp_header.timestamp + + output.data()[output.samples_per_channel_ - 1]), + neteq_->GetPlayoutTimestamp()); + + // Check the timestamp for the last value in the sync buffer. This should + // be one full frame length ahead of the RTP timestamp. + const SyncBuffer* sync_buffer = neteq_->sync_buffer_for_test(); + ASSERT_TRUE(sync_buffer != NULL); + EXPECT_EQ(rtp_header.timestamp + kPayloadLengthSamples, + sync_buffer->end_timestamp()); + + // Check that the number of samples still to play from the sync buffer add + // up with what was already played out. + EXPECT_EQ( + kPayloadLengthSamples - output.data()[output.samples_per_channel_ - 1], + sync_buffer->FutureLength()); +} + +TEST_F(NetEqImplTest, ReorderedPacket) { + UseNoMocks(); + + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + + CreateInstance( + rtc::make_ref_counted(&mock_decoder)); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kSampleRateHz = 8000; + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + rtp_header.extension.hasAudioLevel = true; + rtp_header.extension.audioLevel = 42; + + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, PacketDuration(_, kPayloadLengthBytes)) + .WillRepeatedly(Return(rtc::checked_cast(kPayloadLengthSamples))); + int16_t dummy_output[kPayloadLengthSamples] = {0}; + // The below expectation will make the mock decoder write + // `kPayloadLengthSamples` zeros to the output array, and mark it as speech. + EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(0), kPayloadLengthBytes, + kSampleRateHz, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("L16", 8000, 1))); + + // Insert one packet. + clock_.AdvanceTimeMilliseconds(123456); + Timestamp expected_receive_time = clock_.CurrentTime(); + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + + // Pull audio once. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Verify `output.packet_infos_`. + ASSERT_THAT(output.packet_infos_, SizeIs(1)); + { + const auto& packet_info = output.packet_infos_[0]; + EXPECT_EQ(packet_info.ssrc(), rtp_header.ssrc); + EXPECT_THAT(packet_info.csrcs(), IsEmpty()); + EXPECT_EQ(packet_info.rtp_timestamp(), rtp_header.timestamp); + EXPECT_EQ(packet_info.audio_level(), rtp_header.extension.audioLevel); + EXPECT_EQ(packet_info.receive_time(), expected_receive_time); + } + + // Insert two more packets. The first one is out of order, and is already too + // old, the second one is the expected next packet. + rtp_header.sequenceNumber -= 1; + rtp_header.timestamp -= kPayloadLengthSamples; + rtp_header.extension.audioLevel = 1; + payload[0] = 1; + clock_.AdvanceTimeMilliseconds(1000); + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + rtp_header.sequenceNumber += 2; + rtp_header.timestamp += 2 * kPayloadLengthSamples; + rtp_header.extension.audioLevel = 2; + payload[0] = 2; + clock_.AdvanceTimeMilliseconds(2000); + expected_receive_time = clock_.CurrentTime(); + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + + // Expect only the second packet to be decoded (the one with "2" as the first + // payload byte). + EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(2), kPayloadLengthBytes, + kSampleRateHz, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + + // Pull audio once. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Now check the packet buffer, and make sure it is empty, since the + // out-of-order packet should have been discarded. + EXPECT_TRUE(packet_buffer_->Empty()); + + // NetEq `packets_discarded` should capture this packet discard. + EXPECT_EQ(1u, neteq_->GetLifetimeStatistics().packets_discarded); + + // Verify `output.packet_infos_`. Expect to only see the second packet. + ASSERT_THAT(output.packet_infos_, SizeIs(1)); + { + const auto& packet_info = output.packet_infos_[0]; + EXPECT_EQ(packet_info.ssrc(), rtp_header.ssrc); + EXPECT_THAT(packet_info.csrcs(), IsEmpty()); + EXPECT_EQ(packet_info.rtp_timestamp(), rtp_header.timestamp); + EXPECT_EQ(packet_info.audio_level(), rtp_header.extension.audioLevel); + EXPECT_EQ(packet_info.receive_time(), expected_receive_time); + } + + EXPECT_CALL(mock_decoder, Die()); +} + +// This test verifies that NetEq can handle the situation where the first +// incoming packet is rejected. +TEST_F(NetEqImplTest, FirstPacketUnknown) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kSampleRateHz = 8000; + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Insert one packet. Note that we have not registered any payload type, so + // this packet will be rejected. + EXPECT_EQ(NetEq::kFail, neteq_->InsertPacket(rtp_header, payload)); + + // Pull audio once. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_LE(output.samples_per_channel_, kMaxOutputSize); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kPLC, output.speech_type_); + EXPECT_THAT(output.packet_infos_, IsEmpty()); + + // Register the payload type. + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("l16", 8000, 1))); + + // Insert 10 packets. + for (size_t i = 0; i < 10; ++i) { + rtp_header.sequenceNumber++; + rtp_header.timestamp += kPayloadLengthSamples; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer()); + } + + // Pull audio repeatedly and make sure we get normal output, that is not PLC. + for (size_t i = 0; i < 3; ++i) { + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_LE(output.samples_per_channel_, kMaxOutputSize); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_) + << "NetEq did not decode the packets as expected."; + EXPECT_THAT(output.packet_infos_, SizeIs(1)); + } +} + +// This test verifies that audio interruption is not logged for the initial +// PLC period before the first packet is deocoded. +// TODO(henrik.lundin) Maybe move this test to neteq_network_stats_unittest.cc. +// Make the test parametrized, so that we can test with different initial +// sample rates in NetEq. +class NetEqImplTestSampleRateParameter + : public NetEqImplTest, + public testing::WithParamInterface { + protected: + NetEqImplTestSampleRateParameter() + : NetEqImplTest(), initial_sample_rate_hz_(GetParam()) { + config_.sample_rate_hz = initial_sample_rate_hz_; + } + + const int initial_sample_rate_hz_; +}; + +class NetEqImplTestSdpFormatParameter + : public NetEqImplTest, + public testing::WithParamInterface { + protected: + NetEqImplTestSdpFormatParameter() + : NetEqImplTest(), sdp_format_(GetParam()) {} + const SdpAudioFormat sdp_format_; +}; + +// This test does the following: +// 0. Set up NetEq with initial sample rate given by test parameter, and a codec +// sample rate of 16000. +// 1. Start calling GetAudio before inserting any encoded audio. The audio +// produced will be PLC. +// 2. Insert a number of encoded audio packets. +// 3. Keep calling GetAudio and verify that no audio interruption was logged. +// Call GetAudio until NetEq runs out of data again; PLC starts. +// 4. Insert one more packet. +// 5. Call GetAudio until that packet is decoded and the PLC ends. + +TEST_P(NetEqImplTestSampleRateParameter, + NoAudioInterruptionLoggedBeforeFirstDecode) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kPayloadSampleRateHz = 16000; + const size_t kPayloadLengthSamples = + static_cast(10 * kPayloadSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Register the payload type. + EXPECT_TRUE(neteq_->RegisterPayloadType( + kPayloadType, SdpAudioFormat("l16", kPayloadSampleRateHz, 1))); + + // Pull audio several times. No packets have been inserted yet. + const size_t initial_output_size = + static_cast(10 * initial_sample_rate_hz_ / 1000); // 10 ms + AudioFrame output; + bool muted; + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(initial_output_size, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_NE(AudioFrame::kNormalSpeech, output.speech_type_); + EXPECT_THAT(output.packet_infos_, IsEmpty()); + } + + // Lambda for inserting packets. + auto insert_packet = [&]() { + rtp_header.sequenceNumber++; + rtp_header.timestamp += kPayloadLengthSamples; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + }; + // Insert 10 packets. + for (size_t i = 0; i < 10; ++i) { + insert_packet(); + EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer()); + } + + // Pull audio repeatedly and make sure we get normal output, that is not PLC. + constexpr size_t kOutputSize = + static_cast(10 * kPayloadSampleRateHz / 1000); // 10 ms + for (size_t i = 0; i < 3; ++i) { + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_) + << "NetEq did not decode the packets as expected."; + EXPECT_THAT(output.packet_infos_, SizeIs(1)); + } + + // Verify that no interruption was logged. + auto lifetime_stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(0, lifetime_stats.interruption_count); + + // Keep pulling audio data until a new PLC period is started. + size_t count_loops = 0; + while (output.speech_type_ == AudioFrame::kNormalSpeech) { + // Make sure we don't hang the test if we never go to PLC. + ASSERT_LT(++count_loops, 100u); + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + } + + // Insert a few packets to avoid postpone decoding after expand. + for (size_t i = 0; i < 5; ++i) { + insert_packet(); + } + + // Pull audio until the newly inserted packet is decoded and the PLC ends. + while (output.speech_type_ != AudioFrame::kNormalSpeech) { + // Make sure we don't hang the test if we never go to PLC. + ASSERT_LT(++count_loops, 100u); + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + } + + // Verify that no interruption was logged. + lifetime_stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(0, lifetime_stats.interruption_count); +} + +// This test does the following: +// 0. Set up NetEq with initial sample rate given by test parameter, and a codec +// sample rate of 16000. +// 1. Insert a number of encoded audio packets. +// 2. Call GetAudio and verify that decoded audio is produced. +// 3. Keep calling GetAudio until NetEq runs out of data; PLC starts. +// 4. Keep calling GetAudio until PLC has been produced for at least 150 ms. +// 5. Insert one more packet. +// 6. Call GetAudio until that packet is decoded and the PLC ends. +// 7. Verify that an interruption was logged. + +TEST_P(NetEqImplTestSampleRateParameter, AudioInterruptionLogged) { + UseNoMocks(); + CreateInstance(); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kPayloadSampleRateHz = 16000; + const size_t kPayloadLengthSamples = + static_cast(10 * kPayloadSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + // Register the payload type. + EXPECT_TRUE(neteq_->RegisterPayloadType( + kPayloadType, SdpAudioFormat("l16", kPayloadSampleRateHz, 1))); + + // Lambda for inserting packets. + auto insert_packet = [&]() { + rtp_header.sequenceNumber++; + rtp_header.timestamp += kPayloadLengthSamples; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + }; + // Insert 10 packets. + for (size_t i = 0; i < 10; ++i) { + insert_packet(); + EXPECT_EQ(i + 1, packet_buffer_->NumPacketsInBuffer()); + } + + AudioFrame output; + bool muted; + // Keep pulling audio data until a new PLC period is started. + size_t count_loops = 0; + do { + // Make sure we don't hang the test if we never go to PLC. + ASSERT_LT(++count_loops, 100u); + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + } while (output.speech_type_ == AudioFrame::kNormalSpeech); + + // Pull audio 15 times, which produces 150 ms of output audio. This should + // all be produced as PLC. The total length of the gap will then be 150 ms + // plus an initial fraction of 10 ms at the start and the end of the PLC + // period. In total, less than 170 ms. + for (size_t i = 0; i < 15; ++i) { + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_NE(AudioFrame::kNormalSpeech, output.speech_type_); + } + + // Insert a few packets to avoid postpone decoding after expand. + for (size_t i = 0; i < 5; ++i) { + insert_packet(); + } + + // Pull audio until the newly inserted packet is decoded and the PLC ends. + while (output.speech_type_ != AudioFrame::kNormalSpeech) { + // Make sure we don't hang the test if we never go to PLC. + ASSERT_LT(++count_loops, 100u); + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + } + + // Verify that the interruption was logged. + auto lifetime_stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(1, lifetime_stats.interruption_count); + EXPECT_GT(lifetime_stats.total_interruption_duration_ms, 150); + EXPECT_LT(lifetime_stats.total_interruption_duration_ms, 170); +} + +INSTANTIATE_TEST_SUITE_P(SampleRates, + NetEqImplTestSampleRateParameter, + testing::Values(8000, 16000, 32000, 48000)); + +TEST_P(NetEqImplTestSdpFormatParameter, GetNackListScaledTimestamp) { + UseNoMocks(); + CreateInstance(); + + neteq_->EnableNack(128); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kPayloadSampleRateHz = sdp_format_.clockrate_hz; + const size_t kPayloadLengthSamples = + static_cast(10 * kPayloadSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = kPayloadLengthSamples * 2; + std::vector payload(kPayloadLengthBytes, 0); + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, sdp_format_)); + + auto insert_packet = [&](bool lost = false) { + rtp_header.sequenceNumber++; + rtp_header.timestamp += kPayloadLengthSamples; + if (!lost) + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + }; + + // Insert and decode 10 packets. + for (size_t i = 0; i < 10; ++i) { + insert_packet(); + } + AudioFrame output; + size_t count_loops = 0; + do { + bool muted; + // Make sure we don't hang the test if we never go to PLC. + ASSERT_LT(++count_loops, 100u); + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + } while (output.speech_type_ == AudioFrame::kNormalSpeech); + + insert_packet(); + + insert_packet(/*lost=*/true); + + // Ensure packet gets marked as missing. + for (int i = 0; i < 5; ++i) { + insert_packet(); + } + + // Missing packet recoverable with 5ms RTT. + EXPECT_THAT(neteq_->GetNackList(5), Not(IsEmpty())); + + // No packets should have TimeToPlay > 500ms. + EXPECT_THAT(neteq_->GetNackList(500), IsEmpty()); +} + +INSTANTIATE_TEST_SUITE_P(GetNackList, + NetEqImplTestSdpFormatParameter, + testing::Values(SdpAudioFormat("g722", 8000, 1), + SdpAudioFormat("opus", 48000, 2))); + +// This test verifies that NetEq can handle comfort noise and enters/quits codec +// internal CNG mode properly. +TEST_F(NetEqImplTest, CodecInternalCng) { + UseNoMocks(); + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + CreateInstance( + rtc::make_ref_counted(&mock_decoder)); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kSampleRateKhz = 48; + const size_t kPayloadLengthSamples = + static_cast(20 * kSampleRateKhz); // 20 ms. + const size_t kPayloadLengthBytes = 10; + uint8_t payload[kPayloadLengthBytes] = {0}; + int16_t dummy_output[kPayloadLengthSamples] = {0}; + + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateKhz * 1000)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, PacketDuration(_, kPayloadLengthBytes)) + .WillRepeatedly(Return(rtc::checked_cast(kPayloadLengthSamples))); + // Packed duration when asking the decoder for more CNG data (without a new + // packet). + EXPECT_CALL(mock_decoder, PacketDuration(nullptr, 0)) + .WillRepeatedly(Return(rtc::checked_cast(kPayloadLengthSamples))); + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("opus", 48000, 2))); + + struct Packet { + int sequence_number_delta; + int timestamp_delta; + AudioDecoder::SpeechType decoder_output_type; + }; + std::vector packets = { + {0, 0, AudioDecoder::kSpeech}, + {1, kPayloadLengthSamples, AudioDecoder::kComfortNoise}, + {2, 2 * kPayloadLengthSamples, AudioDecoder::kSpeech}, + {1, kPayloadLengthSamples, AudioDecoder::kSpeech}}; + + for (size_t i = 0; i < packets.size(); ++i) { + rtp_header.sequenceNumber += packets[i].sequence_number_delta; + rtp_header.timestamp += packets[i].timestamp_delta; + payload[0] = i; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + + // Pointee(x) verifies that first byte of the payload equals x, this makes + // it possible to verify that the correct payload is fed to Decode(). + EXPECT_CALL(mock_decoder, DecodeInternal(Pointee(i), kPayloadLengthBytes, + kSampleRateKhz * 1000, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>( + dummy_output, dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(packets[i].decoder_output_type), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + } + + // Expect comfort noise to be returned by the decoder. + EXPECT_CALL(mock_decoder, + DecodeInternal(IsNull(), 0, kSampleRateKhz * 1000, _, _)) + .WillOnce(DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples), + SetArgPointee<4>(AudioDecoder::kComfortNoise), + Return(rtc::checked_cast(kPayloadLengthSamples)))); + + std::vector expected_output = { + AudioFrame::kNormalSpeech, AudioFrame::kCNG, AudioFrame::kNormalSpeech}; + size_t output_index = 0; + + int timeout_counter = 0; + while (!packet_buffer_->Empty()) { + ASSERT_LT(timeout_counter++, 20) << "Test timed out"; + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + if (output_index + 1 < expected_output.size() && + output.speech_type_ == expected_output[output_index + 1]) { + ++output_index; + } else { + EXPECT_EQ(output.speech_type_, expected_output[output_index]); + } + } + + EXPECT_CALL(mock_decoder, Die()); +} + +TEST_F(NetEqImplTest, UnsupportedDecoder) { + UseNoMocks(); + ::testing::NiceMock decoder; + + CreateInstance( + rtc::make_ref_counted(&decoder)); + static const size_t kNetEqMaxFrameSize = 5760; // 120 ms @ 48 kHz. + static const size_t kChannels = 2; + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kSampleRateHz = 8000; + + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = 1; + uint8_t payload[kPayloadLengthBytes] = {0}; + int16_t dummy_output[kPayloadLengthSamples * kChannels] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + const uint8_t kFirstPayloadValue = 1; + const uint8_t kSecondPayloadValue = 2; + + EXPECT_CALL(decoder, + PacketDuration(Pointee(kFirstPayloadValue), kPayloadLengthBytes)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(rtc::checked_cast(kNetEqMaxFrameSize + 1))); + + EXPECT_CALL(decoder, DecodeInternal(Pointee(kFirstPayloadValue), _, _, _, _)) + .Times(0); + + EXPECT_CALL(decoder, DecodeInternal(Pointee(kSecondPayloadValue), + kPayloadLengthBytes, kSampleRateHz, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples * kChannels), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(static_cast(kPayloadLengthSamples * kChannels)))); + + EXPECT_CALL(decoder, + PacketDuration(Pointee(kSecondPayloadValue), kPayloadLengthBytes)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(rtc::checked_cast(kNetEqMaxFrameSize))); + + EXPECT_CALL(decoder, SampleRateHz()).WillRepeatedly(Return(kSampleRateHz)); + + EXPECT_CALL(decoder, Channels()).WillRepeatedly(Return(kChannels)); + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("L16", 8000, 1))); + + // Insert one packet. + payload[0] = kFirstPayloadValue; // This will make Decode() fail. + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + + // Insert another packet. + payload[0] = kSecondPayloadValue; // This will make Decode() successful. + rtp_header.sequenceNumber++; + // The second timestamp needs to be at least 30 ms after the first to make + // the second packet get decoded. + rtp_header.timestamp += 3 * kPayloadLengthSamples; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + + AudioFrame output; + bool muted; + // First call to GetAudio will try to decode the "faulty" packet. + // Expect kFail return value. + EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted)); + // Output size and number of channels should be correct. + const size_t kExpectedOutputSize = 10 * (kSampleRateHz / 1000) * kChannels; + EXPECT_EQ(kExpectedOutputSize, output.samples_per_channel_ * kChannels); + EXPECT_EQ(kChannels, output.num_channels_); + EXPECT_THAT(output.packet_infos_, IsEmpty()); + + // Second call to GetAudio will decode the packet that is ok. No errors are + // expected. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kExpectedOutputSize, output.samples_per_channel_ * kChannels); + EXPECT_EQ(kChannels, output.num_channels_); + EXPECT_THAT(output.packet_infos_, SizeIs(1)); + + // Die isn't called through NiceMock (since it's called by the + // MockAudioDecoder constructor), so it needs to be mocked explicitly. + EXPECT_CALL(decoder, Die()); +} + +// This test inserts packets until the buffer is flushed. After that, it asks +// NetEq for the network statistics. The purpose of the test is to make sure +// that even though the buffer size increment is negative (which it becomes when +// the packet causing a flush is inserted), the packet length stored in the +// decision logic remains valid. +TEST_F(NetEqImplTest, FloodBufferAndGetNetworkStats) { + UseNoMocks(); + CreateInstance(); + + const size_t kPayloadLengthSamples = 80; + const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. + const uint8_t kPayloadType = 17; // Just an arbitrary number. + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("l16", 8000, 1))); + + // Insert packets until the buffer flushes. + for (size_t i = 0; i <= config_.max_packets_in_buffer; ++i) { + EXPECT_EQ(i, packet_buffer_->NumPacketsInBuffer()); + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + rtp_header.timestamp += rtc::checked_cast(kPayloadLengthSamples); + ++rtp_header.sequenceNumber; + } + EXPECT_EQ(1u, packet_buffer_->NumPacketsInBuffer()); + + // Ask for network statistics. This should not crash. + NetEqNetworkStatistics stats; + EXPECT_EQ(NetEq::kOK, neteq_->NetworkStatistics(&stats)); +} + +TEST_F(NetEqImplTest, DecodedPayloadTooShort) { + UseNoMocks(); + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + + CreateInstance( + rtc::make_ref_counted(&mock_decoder)); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kSampleRateHz = 8000; + const size_t kPayloadLengthSamples = + static_cast(10 * kSampleRateHz / 1000); // 10 ms. + const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, PacketDuration(_, _)) + .WillRepeatedly(Return(rtc::checked_cast(kPayloadLengthSamples))); + int16_t dummy_output[kPayloadLengthSamples] = {0}; + // The below expectation will make the mock decoder write + // `kPayloadLengthSamples` - 5 zeros to the output array, and mark it as + // speech. That is, the decoded length is 5 samples shorter than the expected. + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .WillOnce( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kPayloadLengthSamples - 5), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kPayloadLengthSamples - 5)))); + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("L16", 8000, 1))); + + // Insert one packet. + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + + EXPECT_EQ(5u, neteq_->sync_buffer_for_test()->FutureLength()); + + // Pull audio once. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + EXPECT_THAT(output.packet_infos_, SizeIs(1)); + + EXPECT_CALL(mock_decoder, Die()); +} + +// This test checks the behavior of NetEq when audio decoder fails. +TEST_F(NetEqImplTest, DecodingError) { + UseNoMocks(); + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + + CreateInstance( + rtc::make_ref_counted(&mock_decoder)); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kSampleRateHz = 8000; + const int kDecoderErrorCode = -97; // Any negative number. + + // We let decoder return 5 ms each time, and therefore, 2 packets make 10 ms. + const size_t kFrameLengthSamples = + static_cast(5 * kSampleRateHz / 1000); + + const size_t kPayloadLengthBytes = 1; // This can be arbitrary. + + uint8_t payload[kPayloadLengthBytes] = {0}; + + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, PacketDuration(_, _)) + .WillRepeatedly(Return(rtc::checked_cast(kFrameLengthSamples))); + EXPECT_CALL(mock_decoder, ErrorCode()).WillOnce(Return(kDecoderErrorCode)); + EXPECT_CALL(mock_decoder, HasDecodePlc()).WillOnce(Return(false)); + int16_t dummy_output[kFrameLengthSamples] = {0}; + + { + InSequence sequence; // Dummy variable. + // Mock decoder works normally the first time. + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .Times(3) + .WillRepeatedly( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kFrameLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kFrameLengthSamples)))) + .RetiresOnSaturation(); + + // Then mock decoder fails. A common reason for failure can be buffer being + // too short + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .WillOnce(Return(-1)) + .RetiresOnSaturation(); + + // Mock decoder finally returns to normal. + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .Times(2) + .WillRepeatedly( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kFrameLengthSamples), + SetArgPointee<4>(AudioDecoder::kSpeech), + Return(rtc::checked_cast(kFrameLengthSamples)))); + } + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("L16", 8000, 1))); + + // Insert packets. + for (int i = 0; i < 20; ++i) { + rtp_header.sequenceNumber += 1; + rtp_header.timestamp += kFrameLengthSamples; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + } + + // Pull audio. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + EXPECT_THAT(output.packet_infos_, SizeIs(2)); // 5 ms packets vs 10 ms output + + // Pull audio again. Decoder fails. + EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + // We are not expecting anything for output.speech_type_, since an error was + // returned. + + // Pull audio again, should behave normal. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + EXPECT_THAT(output.packet_infos_, SizeIs(2)); // 5 ms packets vs 10 ms output + + EXPECT_CALL(mock_decoder, Die()); +} + +// This test checks the behavior of NetEq when audio decoder fails during CNG. +TEST_F(NetEqImplTest, DecodingErrorDuringInternalCng) { + UseNoMocks(); + + // Create a mock decoder object. + MockAudioDecoder mock_decoder; + CreateInstance( + rtc::make_ref_counted(&mock_decoder)); + + const uint8_t kPayloadType = 17; // Just an arbitrary number. + const int kSampleRateHz = 8000; + const int kDecoderErrorCode = -97; // Any negative number. + + // We let decoder return 5 ms each time, and therefore, 2 packets make 10 ms. + const size_t kFrameLengthSamples = + static_cast(5 * kSampleRateHz / 1000); + + const size_t kPayloadLengthBytes = 1; // This can be arbitrary. + + uint8_t payload[kPayloadLengthBytes] = {0}; + + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_CALL(mock_decoder, Reset()).WillRepeatedly(Return()); + EXPECT_CALL(mock_decoder, SampleRateHz()) + .WillRepeatedly(Return(kSampleRateHz)); + EXPECT_CALL(mock_decoder, Channels()).WillRepeatedly(Return(1)); + EXPECT_CALL(mock_decoder, PacketDuration(_, _)) + .WillRepeatedly(Return(rtc::checked_cast(kFrameLengthSamples))); + EXPECT_CALL(mock_decoder, ErrorCode()).WillOnce(Return(kDecoderErrorCode)); + int16_t dummy_output[kFrameLengthSamples] = {0}; + + { + InSequence sequence; // Dummy variable. + // Mock decoder works normally the first 2 times. + EXPECT_CALL(mock_decoder, + DecodeInternal(_, kPayloadLengthBytes, kSampleRateHz, _, _)) + .Times(2) + .WillRepeatedly( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kFrameLengthSamples), + SetArgPointee<4>(AudioDecoder::kComfortNoise), + Return(rtc::checked_cast(kFrameLengthSamples)))) + .RetiresOnSaturation(); + + // Then mock decoder fails. A common reason for failure can be buffer being + // too short + EXPECT_CALL(mock_decoder, DecodeInternal(nullptr, 0, kSampleRateHz, _, _)) + .WillOnce(Return(-1)) + .RetiresOnSaturation(); + + // Mock decoder finally returns to normal. + EXPECT_CALL(mock_decoder, DecodeInternal(nullptr, 0, kSampleRateHz, _, _)) + .Times(2) + .WillRepeatedly( + DoAll(SetArrayArgument<3>(dummy_output, + dummy_output + kFrameLengthSamples), + SetArgPointee<4>(AudioDecoder::kComfortNoise), + Return(rtc::checked_cast(kFrameLengthSamples)))); + } + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("l16", 8000, 1))); + + // Insert 2 packets. This will make netEq into codec internal CNG mode. + for (int i = 0; i < 2; ++i) { + rtp_header.sequenceNumber += 1; + rtp_header.timestamp += kFrameLengthSamples; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + } + + // Pull audio. + const size_t kMaxOutputSize = static_cast(10 * kSampleRateHz / 1000); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kCNG, output.speech_type_); + + // Pull audio again. Decoder fails. + EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + // We are not expecting anything for output.speech_type_, since an error was + // returned. + + // Pull audio again, should resume codec CNG. + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(kMaxOutputSize, output.samples_per_channel_); + EXPECT_EQ(1u, output.num_channels_); + EXPECT_EQ(AudioFrame::kCNG, output.speech_type_); + + EXPECT_CALL(mock_decoder, Die()); +} + +// Tests that the return value from last_output_sample_rate_hz() is equal to the +// configured inital sample rate. +TEST_F(NetEqImplTest, InitialLastOutputSampleRate) { + UseNoMocks(); + config_.sample_rate_hz = 48000; + CreateInstance(); + EXPECT_EQ(48000, neteq_->last_output_sample_rate_hz()); +} + +TEST_F(NetEqImplTest, TickTimerIncrement) { + UseNoMocks(); + CreateInstance(); + ASSERT_TRUE(tick_timer_); + EXPECT_EQ(0u, tick_timer_->ticks()); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + EXPECT_EQ(1u, tick_timer_->ticks()); +} + +TEST_F(NetEqImplTest, SetBaseMinimumDelay) { + UseNoMocks(); + use_mock_neteq_controller_ = true; + CreateInstance(); + + EXPECT_CALL(*mock_neteq_controller_, SetBaseMinimumDelay(_)) + .WillOnce(Return(true)) + .WillOnce(Return(false)); + + const int delay_ms = 200; + + EXPECT_EQ(true, neteq_->SetBaseMinimumDelayMs(delay_ms)); + EXPECT_EQ(false, neteq_->SetBaseMinimumDelayMs(delay_ms)); +} + +TEST_F(NetEqImplTest, GetBaseMinimumDelayMs) { + UseNoMocks(); + use_mock_neteq_controller_ = true; + CreateInstance(); + + const int delay_ms = 200; + + EXPECT_CALL(*mock_neteq_controller_, GetBaseMinimumDelay()) + .WillOnce(Return(delay_ms)); + + EXPECT_EQ(delay_ms, neteq_->GetBaseMinimumDelayMs()); +} + +TEST_F(NetEqImplTest, TargetDelayMs) { + UseNoMocks(); + use_mock_neteq_controller_ = true; + CreateInstance(); + constexpr int kTargetLevelMs = 510; + EXPECT_CALL(*mock_neteq_controller_, TargetLevelMs()) + .WillOnce(Return(kTargetLevelMs)); + EXPECT_EQ(510, neteq_->TargetDelayMs()); +} + +TEST_F(NetEqImplTest, InsertEmptyPacket) { + UseNoMocks(); + use_mock_neteq_controller_ = true; + CreateInstance(); + + RTPHeader rtp_header; + rtp_header.payloadType = 17; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_CALL(*mock_neteq_controller_, RegisterEmptyPacket()); + neteq_->InsertEmptyPacket(rtp_header); +} + +TEST_F(NetEqImplTest, NotifyControllerOfReorderedPacket) { + using ::testing::AllOf; + using ::testing::Field; + UseNoMocks(); + use_mock_neteq_controller_ = true; + CreateInstance(); + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) + .Times(1) + .WillOnce(Return(NetEq::Operation::kNormal)); + + const int kPayloadLengthSamples = 80; + const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. + const uint8_t kPayloadType = 17; // Just an arbitrary number. + uint8_t payload[kPayloadLengthBytes] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = 0x1234; + rtp_header.timestamp = 0x12345678; + rtp_header.ssrc = 0x87654321; + + EXPECT_TRUE(neteq_->RegisterPayloadType(kPayloadType, + SdpAudioFormat("l16", 8000, 1))); + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + AudioFrame output; + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output, &muted)); + + // Insert second packet that was sent before the first packet. + rtp_header.sequenceNumber -= 1; + rtp_header.timestamp -= kPayloadLengthSamples; + EXPECT_CALL( + *mock_neteq_controller_, + PacketArrived( + /*fs_hz*/ 8000, + /*should_update_stats*/ true, + /*info*/ + AllOf( + Field(&NetEqController::PacketArrivedInfo::packet_length_samples, + kPayloadLengthSamples), + Field(&NetEqController::PacketArrivedInfo::main_sequence_number, + rtp_header.sequenceNumber), + Field(&NetEqController::PacketArrivedInfo::main_timestamp, + rtp_header.timestamp)))); + + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); +} + +// When using a codec with 1000 channels, there should be no crashes. +TEST_F(NetEqImplTest, NoCrashWith1000Channels) { + using ::testing::AllOf; + using ::testing::Field; + UseNoMocks(); + use_mock_decoder_database_ = true; + enable_muted_state_ = true; + CreateInstance(); + const size_t kPayloadLength = 100; + const uint8_t kPayloadType = 0; + const uint16_t kFirstSequenceNumber = 0x1234; + const uint32_t kFirstTimestamp = 0x12345678; + const uint32_t kSsrc = 0x87654321; + uint8_t payload[kPayloadLength] = {0}; + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = kFirstSequenceNumber; + rtp_header.timestamp = kFirstTimestamp; + rtp_header.ssrc = kSsrc; + Packet fake_packet; + fake_packet.payload_type = kPayloadType; + fake_packet.sequence_number = kFirstSequenceNumber; + fake_packet.timestamp = kFirstTimestamp; + + AudioDecoder* decoder = nullptr; + + auto mock_decoder_factory = rtc::make_ref_counted(); + EXPECT_CALL(*mock_decoder_factory, MakeAudioDecoderMock(_, _, _)) + .WillOnce(Invoke([&](const SdpAudioFormat& format, + absl::optional codec_pair_id, + std::unique_ptr* dec) { + EXPECT_EQ("pcmu", format.name); + *dec = std::make_unique(1000); + decoder = dec->get(); + })); + DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1), + absl::nullopt, mock_decoder_factory.get()); + // Expectations for decoder database. + EXPECT_CALL(*mock_decoder_database_, GetDecoderInfo(kPayloadType)) + .WillRepeatedly(Return(&info)); + EXPECT_CALL(*mock_decoder_database_, GetActiveCngDecoder()) + .WillRepeatedly(ReturnNull()); + EXPECT_CALL(*mock_decoder_database_, GetActiveDecoder()) + .WillRepeatedly(Return(decoder)); + EXPECT_CALL(*mock_decoder_database_, SetActiveDecoder(_, _)) + .WillOnce(Invoke([](uint8_t rtp_payload_type, bool* new_decoder) { + *new_decoder = true; + return 0; + })); + + // Insert first packet. + neteq_->InsertPacket(rtp_header, payload); + + AudioFrame audio_frame; + bool muted; + + // Repeat 40 times to ensure we enter muted state. + for (int i = 0; i < 40; i++) { + // GetAudio should return an error, and not crash, even in muted state. + EXPECT_NE(0, neteq_->GetAudio(&audio_frame, &muted)); + } +} + +class Decoder120ms : public AudioDecoder { + public: + Decoder120ms(int sample_rate_hz, SpeechType speech_type) + : sample_rate_hz_(sample_rate_hz), + next_value_(1), + speech_type_(speech_type) {} + + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override { + EXPECT_EQ(sample_rate_hz_, sample_rate_hz); + size_t decoded_len = + rtc::CheckedDivExact(sample_rate_hz, 1000) * 120 * Channels(); + for (size_t i = 0; i < decoded_len; ++i) { + decoded[i] = next_value_++; + } + *speech_type = speech_type_; + return rtc::checked_cast(decoded_len); + } + + void Reset() override { next_value_ = 1; } + int SampleRateHz() const override { return sample_rate_hz_; } + size_t Channels() const override { return 2; } + + private: + int sample_rate_hz_; + int16_t next_value_; + SpeechType speech_type_; +}; + +class NetEqImplTest120ms : public NetEqImplTest { + protected: + NetEqImplTest120ms() : NetEqImplTest() {} + virtual ~NetEqImplTest120ms() {} + + void CreateInstanceNoMocks() { + UseNoMocks(); + CreateInstance(decoder_factory_); + EXPECT_TRUE(neteq_->RegisterPayloadType( + kPayloadType, SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}}))); + } + + void CreateInstanceWithDelayManagerMock() { + UseNoMocks(); + use_mock_neteq_controller_ = true; + CreateInstance(decoder_factory_); + EXPECT_TRUE(neteq_->RegisterPayloadType( + kPayloadType, SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}}))); + } + + uint32_t timestamp_diff_between_packets() const { + return rtc::CheckedDivExact(kSamplingFreq_, 1000u) * 120; + } + + uint32_t first_timestamp() const { return 10u; } + + void GetFirstPacket() { + bool muted; + for (int i = 0; i < 12; i++) { + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_FALSE(muted); + } + } + + void InsertPacket(uint32_t timestamp) { + RTPHeader rtp_header; + rtp_header.payloadType = kPayloadType; + rtp_header.sequenceNumber = sequence_number_; + rtp_header.timestamp = timestamp; + rtp_header.ssrc = 15; + const size_t kPayloadLengthBytes = 1; // This can be arbitrary. + uint8_t payload[kPayloadLengthBytes] = {0}; + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); + sequence_number_++; + } + + void Register120msCodec(AudioDecoder::SpeechType speech_type) { + const uint32_t sampling_freq = kSamplingFreq_; + decoder_factory_ = rtc::make_ref_counted( + [sampling_freq, speech_type]() { + std::unique_ptr decoder = + std::make_unique(sampling_freq, speech_type); + RTC_CHECK_EQ(2, decoder->Channels()); + return decoder; + }); + } + + rtc::scoped_refptr decoder_factory_; + AudioFrame output_; + const uint32_t kPayloadType = 17; + const uint32_t kSamplingFreq_ = 48000; + uint16_t sequence_number_ = 1; +}; + +TEST_F(NetEqImplTest120ms, CodecInternalCng) { + Register120msCodec(AudioDecoder::kComfortNoise); + CreateInstanceNoMocks(); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(NetEq::Operation::kCodecInternalCng, + neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, Normal) { + Register120msCodec(AudioDecoder::kSpeech); + CreateInstanceNoMocks(); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + EXPECT_EQ(NetEq::Operation::kNormal, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, Merge) { + Register120msCodec(AudioDecoder::kSpeech); + CreateInstanceWithDelayManagerMock(); + + EXPECT_CALL(*mock_neteq_controller_, CngOff()).WillRepeatedly(Return(true)); + InsertPacket(first_timestamp()); + + GetFirstPacket(); + bool muted; + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) + .WillOnce(Return(NetEq::Operation::kExpand)); + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + + InsertPacket(first_timestamp() + 2 * timestamp_diff_between_packets()); + + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) + .WillOnce(Return(NetEq::Operation::kMerge)); + + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(NetEq::Operation::kMerge, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, Expand) { + Register120msCodec(AudioDecoder::kSpeech); + CreateInstanceNoMocks(); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(NetEq::Operation::kExpand, neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, FastAccelerate) { + Register120msCodec(AudioDecoder::kSpeech); + CreateInstanceWithDelayManagerMock(); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + InsertPacket(first_timestamp() + timestamp_diff_between_packets()); + + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) + .Times(1) + .WillOnce(Return(NetEq::Operation::kFastAccelerate)); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(NetEq::Operation::kFastAccelerate, + neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, PreemptiveExpand) { + Register120msCodec(AudioDecoder::kSpeech); + CreateInstanceWithDelayManagerMock(); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + InsertPacket(first_timestamp() + timestamp_diff_between_packets()); + + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) + .Times(1) + .WillOnce(Return(NetEq::Operation::kPreemptiveExpand)); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(NetEq::Operation::kPreemptiveExpand, + neteq_->last_operation_for_test()); +} + +TEST_F(NetEqImplTest120ms, Accelerate) { + Register120msCodec(AudioDecoder::kSpeech); + CreateInstanceWithDelayManagerMock(); + + InsertPacket(first_timestamp()); + GetFirstPacket(); + + InsertPacket(first_timestamp() + timestamp_diff_between_packets()); + + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) + .Times(1) + .WillOnce(Return(NetEq::Operation::kAccelerate)); + + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); + EXPECT_EQ(NetEq::Operation::kAccelerate, neteq_->last_operation_for_test()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc new file mode 100644 index 0000000000..a669ad727e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "absl/memory/memory.h" +#include "api/audio/audio_frame.h" +#include "api/audio_codecs/audio_decoder.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/neteq/neteq.h" +#include "modules/audio_coding/neteq/default_neteq_factory.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "system_wrappers/include/clock.h" +#include "test/audio_decoder_proxy_factory.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +namespace { + +std::unique_ptr CreateNetEq( + const NetEq::Config& config, + Clock* clock, + const rtc::scoped_refptr& decoder_factory) { + return DefaultNetEqFactory().CreateNetEq(config, decoder_factory, clock); +} + +} // namespace + +using ::testing::_; +using ::testing::Return; +using ::testing::SetArgPointee; + +class MockAudioDecoder final : public AudioDecoder { + public: + static const int kPacketDuration = 960; // 48 kHz * 20 ms + + MockAudioDecoder(int sample_rate_hz, size_t num_channels) + : sample_rate_hz_(sample_rate_hz), + num_channels_(num_channels), + fec_enabled_(false) {} + ~MockAudioDecoder() override { Die(); } + MOCK_METHOD(void, Die, ()); + + MOCK_METHOD(void, Reset, (), (override)); + + class MockFrame : public AudioDecoder::EncodedAudioFrame { + public: + MockFrame(size_t num_channels) : num_channels_(num_channels) {} + + size_t Duration() const override { return kPacketDuration; } + + absl::optional Decode( + rtc::ArrayView decoded) const override { + const size_t output_size = + sizeof(int16_t) * kPacketDuration * num_channels_; + if (decoded.size() >= output_size) { + memset(decoded.data(), 0, + sizeof(int16_t) * kPacketDuration * num_channels_); + return DecodeResult{kPacketDuration * num_channels_, kSpeech}; + } else { + ADD_FAILURE() << "Expected decoded.size() to be >= output_size (" + << decoded.size() << " vs. " << output_size << ")"; + return absl::nullopt; + } + } + + private: + const size_t num_channels_; + }; + + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override { + std::vector results; + if (fec_enabled_) { + std::unique_ptr fec_frame(new MockFrame(num_channels_)); + results.emplace_back(timestamp - kPacketDuration, 1, + std::move(fec_frame)); + } + + std::unique_ptr frame(new MockFrame(num_channels_)); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; + } + + int PacketDuration(const uint8_t* encoded, + size_t encoded_len) const override { + ADD_FAILURE() << "Since going through ParsePayload, PacketDuration should " + "never get called."; + return kPacketDuration; + } + + bool PacketHasFec(const uint8_t* encoded, size_t encoded_len) const override { + ADD_FAILURE() << "Since going through ParsePayload, PacketHasFec should " + "never get called."; + return fec_enabled_; + } + + int SampleRateHz() const override { return sample_rate_hz_; } + + size_t Channels() const override { return num_channels_; } + + void set_fec_enabled(bool enable_fec) { fec_enabled_ = enable_fec; } + + bool fec_enabled() const { return fec_enabled_; } + + protected: + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override { + ADD_FAILURE() << "Since going through ParsePayload, DecodeInternal should " + "never get called."; + return -1; + } + + private: + const int sample_rate_hz_; + const size_t num_channels_; + bool fec_enabled_; +}; + +class NetEqNetworkStatsTest { + public: + static const int kPayloadSizeByte = 30; + static const int kFrameSizeMs = 20; + static const uint8_t kPayloadType = 95; + static const int kOutputLengthMs = 10; + + enum logic { + kIgnore, + kEqual, + kSmallerThan, + kLargerThan, + }; + + struct NetEqNetworkStatsCheck { + logic current_buffer_size_ms; + logic preferred_buffer_size_ms; + logic jitter_peaks_found; + logic packet_loss_rate; + logic expand_rate; + logic speech_expand_rate; + logic preemptive_rate; + logic accelerate_rate; + logic secondary_decoded_rate; + logic secondary_discarded_rate; + logic added_zero_samples; + NetEqNetworkStatistics stats_ref; + }; + + NetEqNetworkStatsTest(const SdpAudioFormat& format, MockAudioDecoder* decoder) + : decoder_(decoder), + decoder_factory_( + rtc::make_ref_counted(decoder)), + samples_per_ms_(format.clockrate_hz / 1000), + frame_size_samples_(kFrameSizeMs * samples_per_ms_), + rtp_generator_(new RtpGenerator(samples_per_ms_)), + last_lost_time_(0), + packet_loss_interval_(0xffffffff) { + NetEq::Config config; + config.sample_rate_hz = format.clockrate_hz; + neteq_ = CreateNetEq(config, Clock::GetRealTimeClock(), decoder_factory_); + neteq_->RegisterPayloadType(kPayloadType, format); + } + + bool Lost(uint32_t send_time) { + if (send_time - last_lost_time_ >= packet_loss_interval_) { + last_lost_time_ = send_time; + return true; + } + return false; + } + + void SetPacketLossRate(double loss_rate) { + packet_loss_interval_ = + (loss_rate >= 1e-3 ? static_cast(kFrameSizeMs) / loss_rate + : 0xffffffff); + } + + // `stats_ref` + // expects.x = -1, do not care + // expects.x = 0, 'x' in current stats should equal 'x' in `stats_ref` + // expects.x = 1, 'x' in current stats should < 'x' in `stats_ref` + // expects.x = 2, 'x' in current stats should > 'x' in `stats_ref` + void CheckNetworkStatistics(NetEqNetworkStatsCheck expects) { + NetEqNetworkStatistics stats; + neteq_->NetworkStatistics(&stats); + +#define CHECK_NETEQ_NETWORK_STATS(x) \ + switch (expects.x) { \ + case kEqual: \ + EXPECT_EQ(stats.x, expects.stats_ref.x); \ + break; \ + case kSmallerThan: \ + EXPECT_LT(stats.x, expects.stats_ref.x); \ + break; \ + case kLargerThan: \ + EXPECT_GT(stats.x, expects.stats_ref.x); \ + break; \ + default: \ + break; \ + } + + CHECK_NETEQ_NETWORK_STATS(current_buffer_size_ms); + CHECK_NETEQ_NETWORK_STATS(preferred_buffer_size_ms); + CHECK_NETEQ_NETWORK_STATS(jitter_peaks_found); + CHECK_NETEQ_NETWORK_STATS(expand_rate); + CHECK_NETEQ_NETWORK_STATS(speech_expand_rate); + CHECK_NETEQ_NETWORK_STATS(preemptive_rate); + CHECK_NETEQ_NETWORK_STATS(accelerate_rate); + CHECK_NETEQ_NETWORK_STATS(secondary_decoded_rate); + CHECK_NETEQ_NETWORK_STATS(secondary_discarded_rate); + +#undef CHECK_NETEQ_NETWORK_STATS + } + + void RunTest(int num_loops, NetEqNetworkStatsCheck expects) { + uint32_t time_now; + uint32_t next_send_time; + + // Initiate `last_lost_time_`. + time_now = next_send_time = last_lost_time_ = rtp_generator_->GetRtpHeader( + kPayloadType, frame_size_samples_, &rtp_header_); + for (int k = 0; k < num_loops; ++k) { + // Delay by one frame such that the FEC can come in. + while (time_now + kFrameSizeMs >= next_send_time) { + next_send_time = rtp_generator_->GetRtpHeader( + kPayloadType, frame_size_samples_, &rtp_header_); + if (!Lost(next_send_time)) { + static const uint8_t payload[kPayloadSizeByte] = {0}; + ASSERT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header_, payload)); + } + } + bool muted = true; + EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_frame_, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(decoder_->Channels(), output_frame_.num_channels_); + EXPECT_EQ(static_cast(kOutputLengthMs * samples_per_ms_), + output_frame_.samples_per_channel_); + EXPECT_EQ(48000, neteq_->last_output_sample_rate_hz()); + + time_now += kOutputLengthMs; + } + CheckNetworkStatistics(expects); + neteq_->FlushBuffers(); + } + + void DecodeFecTest() { + decoder_->set_fec_enabled(false); + NetEqNetworkStatsCheck expects = {kIgnore, // current_buffer_size_ms + kIgnore, // preferred_buffer_size_ms + kIgnore, // jitter_peaks_found + kEqual, // packet_loss_rate + kEqual, // expand_rate + kEqual, // voice_expand_rate + kIgnore, // preemptive_rate + kEqual, // accelerate_rate + kEqual, // decoded_fec_rate + kEqual, // discarded_fec_rate + kEqual, // added_zero_samples + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + RunTest(50, expects); + + // Next we introduce packet losses. + SetPacketLossRate(0.1); + expects.stats_ref.expand_rate = expects.stats_ref.speech_expand_rate = 898; + RunTest(50, expects); + + // Next we enable FEC. + decoder_->set_fec_enabled(true); + // If FEC fills in the lost packets, no packet loss will be counted. + expects.stats_ref.expand_rate = expects.stats_ref.speech_expand_rate = 0; + expects.stats_ref.secondary_decoded_rate = 2006; + expects.stats_ref.secondary_discarded_rate = 14336; + RunTest(50, expects); + } + + void NoiseExpansionTest() { + NetEqNetworkStatsCheck expects = {kIgnore, // current_buffer_size_ms + kIgnore, // preferred_buffer_size_ms + kIgnore, // jitter_peaks_found + kEqual, // packet_loss_rate + kEqual, // expand_rate + kEqual, // speech_expand_rate + kIgnore, // preemptive_rate + kEqual, // accelerate_rate + kEqual, // decoded_fec_rate + kEqual, // discard_fec_rate + kEqual, // added_zero_samples + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + RunTest(50, expects); + + SetPacketLossRate(1); + expects.stats_ref.expand_rate = 16384; + expects.stats_ref.speech_expand_rate = 5324; + RunTest(10, expects); // Lost 10 * 20ms in a row. + } + + private: + MockAudioDecoder* decoder_; + rtc::scoped_refptr decoder_factory_; + std::unique_ptr neteq_; + + const int samples_per_ms_; + const size_t frame_size_samples_; + std::unique_ptr rtp_generator_; + RTPHeader rtp_header_; + uint32_t last_lost_time_; + uint32_t packet_loss_interval_; + AudioFrame output_frame_; +}; + +TEST(NetEqNetworkStatsTest, DecodeFec) { + MockAudioDecoder decoder(48000, 1); + NetEqNetworkStatsTest test(SdpAudioFormat("opus", 48000, 2), &decoder); + test.DecodeFecTest(); + EXPECT_CALL(decoder, Die()).Times(1); +} + +TEST(NetEqNetworkStatsTest, StereoDecodeFec) { + MockAudioDecoder decoder(48000, 2); + NetEqNetworkStatsTest test(SdpAudioFormat("opus", 48000, 2), &decoder); + test.DecodeFecTest(); + EXPECT_CALL(decoder, Die()).Times(1); +} + +TEST(NetEqNetworkStatsTest, NoiseExpansionTest) { + MockAudioDecoder decoder(48000, 1); + NetEqNetworkStatsTest test(SdpAudioFormat("opus", 48000, 2), &decoder); + test.NoiseExpansionTest(); + EXPECT_CALL(decoder, Die()).Times(1); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_stereo_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_stereo_unittest.cc new file mode 100644 index 0000000000..6fa56fd1c1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_stereo_unittest.cc @@ -0,0 +1,424 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Test to verify correct stereo and multi-channel operation. + +#include +#include +#include +#include + +#include "api/audio/audio_frame.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/neteq/neteq.h" +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/neteq/default_neteq_factory.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +struct TestParameters { + int frame_size; + int sample_rate; + size_t num_channels; +}; + +// This is a parameterized test. The test parameters are supplied through a +// TestParameters struct, which is obtained through the GetParam() method. +// +// The objective of the test is to create a mono input signal and a +// multi-channel input signal, where each channel is identical to the mono +// input channel. The two input signals are processed through their respective +// NetEq instances. After that, the output signals are compared. The expected +// result is that each channel in the multi-channel output is identical to the +// mono output. +class NetEqStereoTest : public ::testing::TestWithParam { + protected: + static const int kTimeStepMs = 10; + static const size_t kMaxBlockSize = 480; // 10 ms @ 48 kHz. + static const uint8_t kPayloadTypeMono = 95; + static const uint8_t kPayloadTypeMulti = 96; + + NetEqStereoTest() + : num_channels_(GetParam().num_channels), + sample_rate_hz_(GetParam().sample_rate), + samples_per_ms_(sample_rate_hz_ / 1000), + frame_size_ms_(GetParam().frame_size), + frame_size_samples_( + static_cast(frame_size_ms_ * samples_per_ms_)), + output_size_samples_(10 * samples_per_ms_), + clock_(0), + rtp_generator_mono_(samples_per_ms_), + rtp_generator_(samples_per_ms_), + payload_size_bytes_(0), + multi_payload_size_bytes_(0), + last_send_time_(0), + last_arrival_time_(0) { + NetEq::Config config; + config.sample_rate_hz = sample_rate_hz_; + DefaultNetEqFactory neteq_factory; + auto decoder_factory = CreateBuiltinAudioDecoderFactory(); + neteq_mono_ = neteq_factory.CreateNetEq(config, decoder_factory, &clock_); + neteq_ = neteq_factory.CreateNetEq(config, decoder_factory, &clock_); + input_ = new int16_t[frame_size_samples_]; + encoded_ = new uint8_t[2 * frame_size_samples_]; + input_multi_channel_ = new int16_t[frame_size_samples_ * num_channels_]; + encoded_multi_channel_ = + new uint8_t[frame_size_samples_ * 2 * num_channels_]; + } + + ~NetEqStereoTest() { + delete[] input_; + delete[] encoded_; + delete[] input_multi_channel_; + delete[] encoded_multi_channel_; + } + + virtual void SetUp() { + const std::string file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + input_file_.reset(new test::InputAudioFile(file_name)); + RTC_CHECK_GE(num_channels_, 2); + ASSERT_TRUE(neteq_mono_->RegisterPayloadType( + kPayloadTypeMono, SdpAudioFormat("l16", sample_rate_hz_, 1))); + ASSERT_TRUE(neteq_->RegisterPayloadType( + kPayloadTypeMulti, + SdpAudioFormat("l16", sample_rate_hz_, num_channels_))); + } + + virtual void TearDown() {} + + int GetNewPackets() { + if (!input_file_->Read(frame_size_samples_, input_)) { + return -1; + } + payload_size_bytes_ = + WebRtcPcm16b_Encode(input_, frame_size_samples_, encoded_); + if (frame_size_samples_ * 2 != payload_size_bytes_) { + return -1; + } + int next_send_time = rtp_generator_mono_.GetRtpHeader( + kPayloadTypeMono, frame_size_samples_, &rtp_header_mono_); + MakeMultiChannelInput(); + multi_payload_size_bytes_ = WebRtcPcm16b_Encode( + input_multi_channel_, frame_size_samples_ * num_channels_, + encoded_multi_channel_); + if (frame_size_samples_ * 2 * num_channels_ != multi_payload_size_bytes_) { + return -1; + } + rtp_generator_.GetRtpHeader(kPayloadTypeMulti, frame_size_samples_, + &rtp_header_); + return next_send_time; + } + + virtual void MakeMultiChannelInput() { + test::InputAudioFile::DuplicateInterleaved( + input_, frame_size_samples_, num_channels_, input_multi_channel_); + } + + virtual void VerifyOutput(size_t num_samples) { + const int16_t* output_data = output_.data(); + const int16_t* output_multi_channel_data = output_multi_channel_.data(); + for (size_t i = 0; i < num_samples; ++i) { + for (size_t j = 0; j < num_channels_; ++j) { + ASSERT_EQ(output_data[i], + output_multi_channel_data[i * num_channels_ + j]) + << "Diff in sample " << i << ", channel " << j << "."; + } + } + } + + virtual int GetArrivalTime(int send_time) { + int arrival_time = last_arrival_time_ + (send_time - last_send_time_); + last_send_time_ = send_time; + last_arrival_time_ = arrival_time; + return arrival_time; + } + + virtual bool Lost() { return false; } + + void RunTest(int num_loops) { + // Get next input packets (mono and multi-channel). + int next_send_time; + int next_arrival_time; + do { + next_send_time = GetNewPackets(); + ASSERT_NE(-1, next_send_time); + next_arrival_time = GetArrivalTime(next_send_time); + } while (Lost()); // If lost, immediately read the next packet. + + int time_now = 0; + for (int k = 0; k < num_loops; ++k) { + while (time_now >= next_arrival_time) { + // Insert packet in mono instance. + ASSERT_EQ(NetEq::kOK, + neteq_mono_->InsertPacket( + rtp_header_mono_, rtc::ArrayView( + encoded_, payload_size_bytes_))); + // Insert packet in multi-channel instance. + ASSERT_EQ(NetEq::kOK, neteq_->InsertPacket( + rtp_header_, rtc::ArrayView( + encoded_multi_channel_, + multi_payload_size_bytes_))); + // Get next input packets (mono and multi-channel). + do { + next_send_time = GetNewPackets(); + ASSERT_NE(-1, next_send_time); + next_arrival_time = GetArrivalTime(next_send_time); + } while (Lost()); // If lost, immediately read the next packet. + } + // Get audio from mono instance. + bool muted; + EXPECT_EQ(NetEq::kOK, neteq_mono_->GetAudio(&output_, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(1u, output_.num_channels_); + EXPECT_EQ(output_size_samples_, output_.samples_per_channel_); + // Get audio from multi-channel instance. + ASSERT_EQ(NetEq::kOK, neteq_->GetAudio(&output_multi_channel_, &muted)); + ASSERT_FALSE(muted); + EXPECT_EQ(num_channels_, output_multi_channel_.num_channels_); + EXPECT_EQ(output_size_samples_, + output_multi_channel_.samples_per_channel_); + rtc::StringBuilder ss; + ss << "Lap number " << k << "."; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + // Compare mono and multi-channel. + ASSERT_NO_FATAL_FAILURE(VerifyOutput(output_size_samples_)); + + time_now += kTimeStepMs; + clock_.AdvanceTimeMilliseconds(kTimeStepMs); + } + } + + const size_t num_channels_; + const int sample_rate_hz_; + const int samples_per_ms_; + const int frame_size_ms_; + const size_t frame_size_samples_; + const size_t output_size_samples_; + SimulatedClock clock_; + std::unique_ptr neteq_mono_; + std::unique_ptr neteq_; + test::RtpGenerator rtp_generator_mono_; + test::RtpGenerator rtp_generator_; + int16_t* input_; + int16_t* input_multi_channel_; + uint8_t* encoded_; + uint8_t* encoded_multi_channel_; + AudioFrame output_; + AudioFrame output_multi_channel_; + RTPHeader rtp_header_mono_; + RTPHeader rtp_header_; + size_t payload_size_bytes_; + size_t multi_payload_size_bytes_; + int last_send_time_; + int last_arrival_time_; + std::unique_ptr input_file_; +}; + +class NetEqStereoTestNoJitter : public NetEqStereoTest { + protected: + NetEqStereoTestNoJitter() : NetEqStereoTest() { + // Start the sender 100 ms before the receiver to pre-fill the buffer. + // This is to avoid doing preemptive expand early in the test. + // TODO(hlundin): Mock the decision making instead to control the modes. + last_arrival_time_ = -100; + } +}; + +TEST_P(NetEqStereoTestNoJitter, RunTest) { + RunTest(8); +} + +class NetEqStereoTestPositiveDrift : public NetEqStereoTest { + protected: + NetEqStereoTestPositiveDrift() : NetEqStereoTest(), drift_factor(0.9) { + // Start the sender 100 ms before the receiver to pre-fill the buffer. + // This is to avoid doing preemptive expand early in the test. + // TODO(hlundin): Mock the decision making instead to control the modes. + last_arrival_time_ = -100; + } + virtual int GetArrivalTime(int send_time) { + int arrival_time = + last_arrival_time_ + drift_factor * (send_time - last_send_time_); + last_send_time_ = send_time; + last_arrival_time_ = arrival_time; + return arrival_time; + } + + double drift_factor; +}; + +TEST_P(NetEqStereoTestPositiveDrift, RunTest) { + RunTest(100); +} + +class NetEqStereoTestNegativeDrift : public NetEqStereoTestPositiveDrift { + protected: + NetEqStereoTestNegativeDrift() : NetEqStereoTestPositiveDrift() { + drift_factor = 1.1; + last_arrival_time_ = 0; + } +}; + +TEST_P(NetEqStereoTestNegativeDrift, RunTest) { + RunTest(100); +} + +class NetEqStereoTestDelays : public NetEqStereoTest { + protected: + static const int kDelayInterval = 10; + static const int kDelay = 1000; + NetEqStereoTestDelays() : NetEqStereoTest(), frame_index_(0) {} + + virtual int GetArrivalTime(int send_time) { + // Deliver immediately, unless we have a back-log. + int arrival_time = std::min(last_arrival_time_, send_time); + if (++frame_index_ % kDelayInterval == 0) { + // Delay this packet. + arrival_time += kDelay; + } + last_send_time_ = send_time; + last_arrival_time_ = arrival_time; + return arrival_time; + } + + int frame_index_; +}; + +TEST_P(NetEqStereoTestDelays, RunTest) { + RunTest(1000); +} + +class NetEqStereoTestLosses : public NetEqStereoTest { + protected: + static const int kLossInterval = 10; + NetEqStereoTestLosses() : NetEqStereoTest(), frame_index_(0) {} + + virtual bool Lost() { return (++frame_index_) % kLossInterval == 0; } + + // TODO(hlundin): NetEq is not giving bitexact results for these cases. + virtual void VerifyOutput(size_t num_samples) { + for (size_t i = 0; i < num_samples; ++i) { + const int16_t* output_data = output_.data(); + const int16_t* output_multi_channel_data = output_multi_channel_.data(); + auto first_channel_sample = output_multi_channel_data[i * num_channels_]; + for (size_t j = 0; j < num_channels_; ++j) { + const int kErrorMargin = 200; + EXPECT_NEAR(output_data[i], + output_multi_channel_data[i * num_channels_ + j], + kErrorMargin) + << "Diff in sample " << i << ", channel " << j << "."; + EXPECT_EQ(first_channel_sample, + output_multi_channel_data[i * num_channels_ + j]); + } + } + } + + int frame_index_; +}; + +TEST_P(NetEqStereoTestLosses, RunTest) { + RunTest(100); +} + +class NetEqStereoTestSingleActiveChannelPlc : public NetEqStereoTestLosses { + protected: + NetEqStereoTestSingleActiveChannelPlc() : NetEqStereoTestLosses() {} + + virtual void MakeMultiChannelInput() override { + // Create a multi-channel input by copying the mono channel from file to the + // first channel, and setting the others to zero. + memset(input_multi_channel_, 0, + frame_size_samples_ * num_channels_ * sizeof(int16_t)); + for (size_t i = 0; i < frame_size_samples_; ++i) { + input_multi_channel_[i * num_channels_] = input_[i]; + } + } + + virtual void VerifyOutput(size_t num_samples) override { + // Simply verify that all samples in channels other than the first are zero. + const int16_t* output_multi_channel_data = output_multi_channel_.data(); + for (size_t i = 0; i < num_samples; ++i) { + for (size_t j = 1; j < num_channels_; ++j) { + EXPECT_EQ(0, output_multi_channel_data[i * num_channels_ + j]) + << "Sample " << i << ", channel " << j << " is non-zero."; + } + } + } +}; + +TEST_P(NetEqStereoTestSingleActiveChannelPlc, RunTest) { + RunTest(100); +} + +// Creates a list of parameter sets. +std::list GetTestParameters() { + std::list l; + const int sample_rates[] = {8000, 16000, 32000}; + const int num_rates = sizeof(sample_rates) / sizeof(sample_rates[0]); + // Loop through sample rates. + for (int rate_index = 0; rate_index < num_rates; ++rate_index) { + int sample_rate = sample_rates[rate_index]; + // Loop through all frame sizes between 10 and 60 ms. + for (int frame_size = 10; frame_size <= 60; frame_size += 10) { + TestParameters p; + p.frame_size = frame_size; + p.sample_rate = sample_rate; + p.num_channels = 2; + l.push_back(p); + if (sample_rate == 8000) { + // Add a five-channel test for 8000 Hz. + p.num_channels = 5; + l.push_back(p); + } + } + } + return l; +} + +// Pretty-printing the test parameters in case of an error. +void PrintTo(const TestParameters& p, ::std::ostream* os) { + *os << "{frame_size = " << p.frame_size + << ", num_channels = " << p.num_channels + << ", sample_rate = " << p.sample_rate << "}"; +} + +// Instantiate the tests. Each test is instantiated using the function above, +// so that all different parameter combinations are tested. +INSTANTIATE_TEST_SUITE_P(MultiChannel, + NetEqStereoTestNoJitter, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_SUITE_P(MultiChannel, + NetEqStereoTestPositiveDrift, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_SUITE_P(MultiChannel, + NetEqStereoTestNegativeDrift, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_SUITE_P(MultiChannel, + NetEqStereoTestDelays, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_SUITE_P(MultiChannel, + NetEqStereoTestLosses, + ::testing::ValuesIn(GetTestParameters())); + +INSTANTIATE_TEST_SUITE_P(MultiChannel, + NetEqStereoTestSingleActiveChannelPlc, + ::testing::ValuesIn(GetTestParameters())); +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_unittest.cc new file mode 100644 index 0000000000..fff14bf830 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_unittest.cc @@ -0,0 +1,1009 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/neteq/neteq.h" + +#include +#include +#include // memset + +#include +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "api/audio/audio_frame.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/neteq/test/neteq_decoding_test.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "modules/audio_coding/neteq/tools/neteq_packet_source_input.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" +#include "modules/include/module_common_types_public.h" +#include "modules/rtp_rtcp/include/rtcp_statistics.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/message_digest.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/system/arch.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +ABSL_FLAG(bool, gen_ref, false, "Generate reference files."); + +namespace webrtc { + +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) && \ + defined(WEBRTC_NETEQ_UNITTEST_BITEXACT) && \ + (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) && \ + defined(WEBRTC_CODEC_ILBC) +#define MAYBE_TestBitExactness TestBitExactness +#else +#define MAYBE_TestBitExactness DISABLED_TestBitExactness +#endif +TEST_F(NetEqDecodingTest, MAYBE_TestBitExactness) { + const std::string input_rtp_file = + webrtc::test::ResourcePath("audio_coding/neteq_universal_new", "rtp"); + + const std::string output_checksum = + "dee7a10ab92526876a70a85bc48a4906901af3df"; + + const std::string network_stats_checksum = + "911dbf5fd97f48d25b8f0967286eb73c9d6f6158"; + + DecodeAndCompare(input_rtp_file, output_checksum, network_stats_checksum, + absl::GetFlag(FLAGS_gen_ref)); +} + +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) && \ + defined(WEBRTC_NETEQ_UNITTEST_BITEXACT) && defined(WEBRTC_CODEC_OPUS) +#define MAYBE_TestOpusBitExactness TestOpusBitExactness +#else +#define MAYBE_TestOpusBitExactness DISABLED_TestOpusBitExactness +#endif +TEST_F(NetEqDecodingTest, MAYBE_TestOpusBitExactness) { + const std::string input_rtp_file = + webrtc::test::ResourcePath("audio_coding/neteq_opus", "rtp"); + + const std::string output_checksum = + "fec6827bb9ee0b21770bbbb4a3a6f8823bf537dc|" + "3610cc7be4b3407b9c273b1299ab7f8f47cca96b"; + + const std::string network_stats_checksum = + "3d043e47e5f4bb81d37e7bce8c44bf802965c853|" + "076662525572dba753b11578330bd491923f7f5e"; + + DecodeAndCompare(input_rtp_file, output_checksum, network_stats_checksum, + absl::GetFlag(FLAGS_gen_ref)); +} + +#if defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64) && \ + defined(WEBRTC_NETEQ_UNITTEST_BITEXACT) && defined(WEBRTC_CODEC_OPUS) +#define MAYBE_TestOpusDtxBitExactness TestOpusDtxBitExactness +#else +#define MAYBE_TestOpusDtxBitExactness DISABLED_TestOpusDtxBitExactness +#endif +TEST_F(NetEqDecodingTest, MAYBE_TestOpusDtxBitExactness) { + const std::string input_rtp_file = + webrtc::test::ResourcePath("audio_coding/neteq_opus_dtx", "rtp"); + + const std::string output_checksum = + "b3c4899eab5378ef5e54f2302948872149f6ad5e|" + "589e975ec31ea13f302457fea1425be9380ffb96"; + + const std::string network_stats_checksum = + "dc8447b9fee1a21fd5d1f4045d62b982a3fb0215"; + + DecodeAndCompare(input_rtp_file, output_checksum, network_stats_checksum, + absl::GetFlag(FLAGS_gen_ref)); +} + +// Use fax mode to avoid time-scaling. This is to simplify the testing of +// packet waiting times in the packet buffer. +class NetEqDecodingTestFaxMode : public NetEqDecodingTest { + protected: + NetEqDecodingTestFaxMode() : NetEqDecodingTest() { + config_.for_test_no_time_stretching = true; + } + void TestJitterBufferDelay(bool apply_packet_loss); +}; + +TEST_F(NetEqDecodingTestFaxMode, TestFrameWaitingTimeStatistics) { + // Insert 30 dummy packets at once. Each packet contains 10 ms 16 kHz audio. + size_t num_frames = 30; + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + for (size_t i = 0; i < num_frames; ++i) { + const uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + rtp_info.sequenceNumber = rtc::checked_cast(i); + rtp_info.timestamp = rtc::checked_cast(i * kSamples); + rtp_info.ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info.payloadType = 94; // PCM16b WB codec. + rtp_info.markerBit = 0; + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + } + // Pull out all data. + for (size_t i = 0; i < num_frames; ++i) { + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + + NetEqNetworkStatistics stats; + EXPECT_EQ(0, neteq_->NetworkStatistics(&stats)); + // Since all frames are dumped into NetEQ at once, but pulled out with 10 ms + // spacing (per definition), we expect the delay to increase with 10 ms for + // each packet. Thus, we are calculating the statistics for a series from 10 + // to 300, in steps of 10 ms. + EXPECT_EQ(155, stats.mean_waiting_time_ms); + EXPECT_EQ(155, stats.median_waiting_time_ms); + EXPECT_EQ(10, stats.min_waiting_time_ms); + EXPECT_EQ(300, stats.max_waiting_time_ms); + + // Check statistics again and make sure it's been reset. + EXPECT_EQ(0, neteq_->NetworkStatistics(&stats)); + EXPECT_EQ(-1, stats.mean_waiting_time_ms); + EXPECT_EQ(-1, stats.median_waiting_time_ms); + EXPECT_EQ(-1, stats.min_waiting_time_ms); + EXPECT_EQ(-1, stats.max_waiting_time_ms); +} + + +TEST_F(NetEqDecodingTest, LongCngWithNegativeClockDrift) { + // Apply a clock drift of -25 ms / s (sender faster than receiver). + const double kDriftFactor = 1000.0 / (1000.0 + 25.0); + const double kNetworkFreezeTimeMs = 0.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 20; + const int kMaxTimeToSpeechMs = 100; + LongCngWithClockDrift(kDriftFactor, kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithPositiveClockDrift) { + // Apply a clock drift of +25 ms / s (sender slower than receiver). + const double kDriftFactor = 1000.0 / (1000.0 - 25.0); + const double kNetworkFreezeTimeMs = 0.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 40; + const int kMaxTimeToSpeechMs = 100; + LongCngWithClockDrift(kDriftFactor, kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithNegativeClockDriftNetworkFreeze) { + // Apply a clock drift of -25 ms / s (sender faster than receiver). + const double kDriftFactor = 1000.0 / (1000.0 + 25.0); + const double kNetworkFreezeTimeMs = 5000.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 60; + const int kMaxTimeToSpeechMs = 200; + LongCngWithClockDrift(kDriftFactor, kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithPositiveClockDriftNetworkFreeze) { + // Apply a clock drift of +25 ms / s (sender slower than receiver). + const double kDriftFactor = 1000.0 / (1000.0 - 25.0); + const double kNetworkFreezeTimeMs = 5000.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 40; + const int kMaxTimeToSpeechMs = 100; + LongCngWithClockDrift(kDriftFactor, kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithPositiveClockDriftNetworkFreezeExtraPull) { + // Apply a clock drift of +25 ms / s (sender slower than receiver). + const double kDriftFactor = 1000.0 / (1000.0 - 25.0); + const double kNetworkFreezeTimeMs = 5000.0; + const bool kGetAudioDuringFreezeRecovery = true; + const int kDelayToleranceMs = 40; + const int kMaxTimeToSpeechMs = 100; + LongCngWithClockDrift(kDriftFactor, kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, LongCngWithoutClockDrift) { + const double kDriftFactor = 1.0; // No drift. + const double kNetworkFreezeTimeMs = 0.0; + const bool kGetAudioDuringFreezeRecovery = false; + const int kDelayToleranceMs = 10; + const int kMaxTimeToSpeechMs = 50; + LongCngWithClockDrift(kDriftFactor, kNetworkFreezeTimeMs, + kGetAudioDuringFreezeRecovery, kDelayToleranceMs, + kMaxTimeToSpeechMs); +} + +TEST_F(NetEqDecodingTest, UnknownPayloadType) { + const size_t kPayloadBytes = 100; + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + rtp_info.payloadType = 1; // Not registered as a decoder. + EXPECT_EQ(NetEq::kFail, neteq_->InsertPacket(rtp_info, payload)); +} + +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) +#define MAYBE_DecoderError DecoderError +#else +#define MAYBE_DecoderError DISABLED_DecoderError +#endif + +TEST_F(NetEqDecodingTest, MAYBE_DecoderError) { + const size_t kPayloadBytes = 100; + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + rtp_info.payloadType = 103; // iSAC, but the payload is invalid. + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + // Set all of `out_data_` to 1, and verify that it was set to 0 by the call + // to GetAudio. + int16_t* out_frame_data = out_frame_.mutable_data(); + for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; ++i) { + out_frame_data[i] = 1; + } + bool muted; + EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_FALSE(muted); + + // Verify that the first 160 samples are set to 0. + static const int kExpectedOutputLength = 160; // 10 ms at 16 kHz sample rate. + const int16_t* const_out_frame_data = out_frame_.data(); + for (int i = 0; i < kExpectedOutputLength; ++i) { + rtc::StringBuilder ss; + ss << "i = " << i; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + EXPECT_EQ(0, const_out_frame_data[i]); + } +} + +TEST_F(NetEqDecodingTest, GetAudioBeforeInsertPacket) { + // Set all of `out_data_` to 1, and verify that it was set to 0 by the call + // to GetAudio. + int16_t* out_frame_data = out_frame_.mutable_data(); + for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; ++i) { + out_frame_data[i] = 1; + } + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_FALSE(muted); + // Verify that the first block of samples is set to 0. + static const int kExpectedOutputLength = + kInitSampleRateHz / 100; // 10 ms at initial sample rate. + const int16_t* const_out_frame_data = out_frame_.data(); + for (int i = 0; i < kExpectedOutputLength; ++i) { + rtc::StringBuilder ss; + ss << "i = " << i; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + EXPECT_EQ(0, const_out_frame_data[i]); + } + // Verify that the sample rate did not change from the initial configuration. + EXPECT_EQ(config_.sample_rate_hz, neteq_->last_output_sample_rate_hz()); +} + +class NetEqBgnTest : public NetEqDecodingTest { + protected: + void CheckBgn(int sampling_rate_hz) { + size_t expected_samples_per_channel = 0; + uint8_t payload_type = 0xFF; // Invalid. + if (sampling_rate_hz == 8000) { + expected_samples_per_channel = kBlockSize8kHz; + payload_type = 93; // PCM 16, 8 kHz. + } else if (sampling_rate_hz == 16000) { + expected_samples_per_channel = kBlockSize16kHz; + payload_type = 94; // PCM 16, 16 kHZ. + } else if (sampling_rate_hz == 32000) { + expected_samples_per_channel = kBlockSize32kHz; + payload_type = 95; // PCM 16, 32 kHz. + } else { + ASSERT_TRUE(false); // Unsupported test case. + } + + AudioFrame output; + test::AudioLoop input; + // We are using the same 32 kHz input file for all tests, regardless of + // `sampling_rate_hz`. The output may sound weird, but the test is still + // valid. + ASSERT_TRUE(input.Init( + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 10 * sampling_rate_hz, // Max 10 seconds loop length. + expected_samples_per_channel)); + + // Payload of 10 ms of PCM16 32 kHz. + uint8_t payload[kBlockSize32kHz * sizeof(int16_t)]; + RTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + rtp_info.payloadType = payload_type; + + bool muted; + for (int n = 0; n < 10; ++n) { // Insert few packets and get audio. + auto block = input.GetNextBlock(); + ASSERT_EQ(expected_samples_per_channel, block.size()); + size_t enc_len_bytes = + WebRtcPcm16b_Encode(block.data(), block.size(), payload); + ASSERT_EQ(enc_len_bytes, expected_samples_per_channel * 2); + + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView( + payload, enc_len_bytes))); + output.Reset(); + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(1u, output.num_channels_); + ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_); + ASSERT_EQ(AudioFrame::kNormalSpeech, output.speech_type_); + + // Next packet. + rtp_info.timestamp += + rtc::checked_cast(expected_samples_per_channel); + rtp_info.sequenceNumber++; + } + + output.Reset(); + + // Get audio without inserting packets, expecting PLC and PLC-to-CNG. Pull + // one frame without checking speech-type. This is the first frame pulled + // without inserting any packet, and might not be labeled as PLC. + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(1u, output.num_channels_); + ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_); + + // To be able to test the fading of background noise we need at lease to + // pull 611 frames. + const int kFadingThreshold = 611; + + // Test several CNG-to-PLC packet for the expected behavior. The number 20 + // is arbitrary, but sufficiently large to test enough number of frames. + const int kNumPlcToCngTestFrames = 20; + bool plc_to_cng = false; + for (int n = 0; n < kFadingThreshold + kNumPlcToCngTestFrames; ++n) { + output.Reset(); + // Set to non-zero. + memset(output.mutable_data(), 1, AudioFrame::kMaxDataSizeBytes); + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + ASSERT_FALSE(muted); + ASSERT_EQ(1u, output.num_channels_); + ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_); + if (output.speech_type_ == AudioFrame::kPLCCNG) { + plc_to_cng = true; + double sum_squared = 0; + const int16_t* output_data = output.data(); + for (size_t k = 0; + k < output.num_channels_ * output.samples_per_channel_; ++k) + sum_squared += output_data[k] * output_data[k]; + EXPECT_EQ(0, sum_squared); + } else { + EXPECT_EQ(AudioFrame::kPLC, output.speech_type_); + } + } + EXPECT_TRUE(plc_to_cng); // Just to be sure that PLC-to-CNG has occurred. + } +}; + +TEST_F(NetEqBgnTest, RunTest) { + CheckBgn(8000); + CheckBgn(16000); + CheckBgn(32000); +} + +TEST_F(NetEqDecodingTest, SequenceNumberWrap) { + // Start with a sequence number that will soon wrap. + std::set drop_seq_numbers; // Don't drop any packets. + WrapTest(0xFFFF - 10, 0, drop_seq_numbers, true, false); +} + +TEST_F(NetEqDecodingTest, SequenceNumberWrapAndDrop) { + // Start with a sequence number that will soon wrap. + std::set drop_seq_numbers; + drop_seq_numbers.insert(0xFFFF); + drop_seq_numbers.insert(0x0); + WrapTest(0xFFFF - 10, 0, drop_seq_numbers, true, false); +} + +TEST_F(NetEqDecodingTest, TimestampWrap) { + // Start with a timestamp that will soon wrap. + std::set drop_seq_numbers; + WrapTest(0, 0xFFFFFFFF - 3000, drop_seq_numbers, false, true); +} + +TEST_F(NetEqDecodingTest, TimestampAndSequenceNumberWrap) { + // Start with a timestamp and a sequence number that will wrap at the same + // time. + std::set drop_seq_numbers; + WrapTest(0xFFFF - 10, 0xFFFFFFFF - 5000, drop_seq_numbers, true, true); +} + +TEST_F(NetEqDecodingTest, DiscardDuplicateCng) { + uint16_t seq_no = 0; + uint32_t timestamp = 0; + const int kFrameSizeMs = 10; + const int kSampleRateKhz = 16; + const int kSamples = kFrameSizeMs * kSampleRateKhz; + const size_t kPayloadBytes = kSamples * 2; + + const int algorithmic_delay_samples = + std::max(algorithmic_delay_ms_ * kSampleRateKhz, 5 * kSampleRateKhz / 8); + // Insert three speech packets. Three are needed to get the frame length + // correct. + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + bool muted; + for (int i = 0; i < 3; ++i) { + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + ++seq_no; + timestamp += kSamples; + + // Pull audio once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + // Verify speech output. + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); + + // Insert same CNG packet twice. + const int kCngPeriodMs = 100; + const int kCngPeriodSamples = kCngPeriodMs * kSampleRateKhz; + size_t payload_len; + PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len); + // This is the first time this CNG packet is inserted. + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView( + payload, payload_len))); + + // Pull audio once and make sure CNG is played. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + EXPECT_FALSE( + neteq_->GetPlayoutTimestamp()); // Returns empty value during CNG. + EXPECT_EQ(timestamp - algorithmic_delay_samples, + out_frame_.timestamp_ + out_frame_.samples_per_channel_); + + // Insert the same CNG packet again. Note that at this point it is old, since + // we have already decoded the first copy of it. + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView( + payload, payload_len))); + + // Pull audio until we have played `kCngPeriodMs` of CNG. Start at 10 ms since + // we have already pulled out CNG once. + for (int cng_time_ms = 10; cng_time_ms < kCngPeriodMs; cng_time_ms += 10) { + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + EXPECT_FALSE( + neteq_->GetPlayoutTimestamp()); // Returns empty value during CNG. + EXPECT_EQ(timestamp - algorithmic_delay_samples, + out_frame_.timestamp_ + out_frame_.samples_per_channel_); + } + + ++seq_no; + timestamp += kCngPeriodSamples; + uint32_t first_speech_timestamp = timestamp; + // Insert speech again. + for (int i = 0; i < 3; ++i) { + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + ++seq_no; + timestamp += kSamples; + } + + // Pull audio once and verify that the output is speech again. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); + absl::optional playout_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + EXPECT_EQ(first_speech_timestamp + kSamples - algorithmic_delay_samples, + *playout_timestamp); +} + +TEST_F(NetEqDecodingTest, CngFirst) { + uint16_t seq_no = 0; + uint32_t timestamp = 0; + const int kFrameSizeMs = 10; + const int kSampleRateKhz = 16; + const int kSamples = kFrameSizeMs * kSampleRateKhz; + const int kPayloadBytes = kSamples * 2; + const int kCngPeriodMs = 100; + const int kCngPeriodSamples = kCngPeriodMs * kSampleRateKhz; + size_t payload_len; + + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + + PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len); + ASSERT_EQ(NetEq::kOK, + neteq_->InsertPacket( + rtp_info, rtc::ArrayView(payload, payload_len))); + ++seq_no; + timestamp += kCngPeriodSamples; + + // Pull audio once and make sure CNG is played. + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + + // Insert some speech packets. + const uint32_t first_speech_timestamp = timestamp; + int timeout_counter = 0; + do { + ASSERT_LT(timeout_counter++, 20) << "Test timed out"; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + ++seq_no; + timestamp += kSamples; + + // Pull audio once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } while (!IsNewerTimestamp(out_frame_.timestamp_, first_speech_timestamp)); + // Verify speech output. + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); +} + +class NetEqDecodingTestWithMutedState : public NetEqDecodingTest { + public: + NetEqDecodingTestWithMutedState() : NetEqDecodingTest() { + config_.enable_muted_state = true; + } + + protected: + static constexpr size_t kSamples = 10 * 16; + static constexpr size_t kPayloadBytes = kSamples * 2; + + void InsertPacket(uint32_t rtp_timestamp) { + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(0, rtp_timestamp, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + } + + void InsertCngPacket(uint32_t rtp_timestamp) { + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + size_t payload_len; + PopulateCng(0, rtp_timestamp, &rtp_info, payload, &payload_len); + EXPECT_EQ(NetEq::kOK, + neteq_->InsertPacket(rtp_info, rtc::ArrayView( + payload, payload_len))); + } + + bool GetAudioReturnMuted() { + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + return muted; + } + + void GetAudioUntilMuted() { + while (!GetAudioReturnMuted()) { + ASSERT_LT(counter_++, 1000) << "Test timed out"; + } + } + + void GetAudioUntilNormal() { + bool muted = false; + while (out_frame_.speech_type_ != AudioFrame::kNormalSpeech) { + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_LT(counter_++, 1000) << "Test timed out"; + } + EXPECT_FALSE(muted); + } + + int counter_ = 0; +}; + +// Verifies that NetEq goes in and out of muted state as expected. +TEST_F(NetEqDecodingTestWithMutedState, MutedState) { + // Insert one speech packet. + InsertPacket(0); + // Pull out audio once and expect it not to be muted. + EXPECT_FALSE(GetAudioReturnMuted()); + // Pull data until faded out. + GetAudioUntilMuted(); + EXPECT_TRUE(out_frame_.muted()); + + // Verify that output audio is not written during muted mode. Other parameters + // should be correct, though. + AudioFrame new_frame; + int16_t* frame_data = new_frame.mutable_data(); + for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; i++) { + frame_data[i] = 17; + } + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&new_frame, &muted)); + EXPECT_TRUE(muted); + EXPECT_TRUE(out_frame_.muted()); + for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; i++) { + EXPECT_EQ(17, frame_data[i]); + } + EXPECT_EQ(out_frame_.timestamp_ + out_frame_.samples_per_channel_, + new_frame.timestamp_); + EXPECT_EQ(out_frame_.samples_per_channel_, new_frame.samples_per_channel_); + EXPECT_EQ(out_frame_.sample_rate_hz_, new_frame.sample_rate_hz_); + EXPECT_EQ(out_frame_.num_channels_, new_frame.num_channels_); + EXPECT_EQ(out_frame_.speech_type_, new_frame.speech_type_); + EXPECT_EQ(out_frame_.vad_activity_, new_frame.vad_activity_); + + // Insert new data. Timestamp is corrected for the time elapsed since the last + // packet. Verify that normal operation resumes. + InsertPacket(kSamples * counter_); + GetAudioUntilNormal(); + EXPECT_FALSE(out_frame_.muted()); + + NetEqNetworkStatistics stats; + EXPECT_EQ(0, neteq_->NetworkStatistics(&stats)); + // NetEqNetworkStatistics::expand_rate tells the fraction of samples that were + // concealment samples, in Q14 (16384 = 100%) .The vast majority should be + // concealment samples in this test. + EXPECT_GT(stats.expand_rate, 14000); + // And, it should be greater than the speech_expand_rate. + EXPECT_GT(stats.expand_rate, stats.speech_expand_rate); +} + +// Verifies that NetEq goes out of muted state when given a delayed packet. +TEST_F(NetEqDecodingTestWithMutedState, MutedStateDelayedPacket) { + // Insert one speech packet. + InsertPacket(0); + // Pull out audio once and expect it not to be muted. + EXPECT_FALSE(GetAudioReturnMuted()); + // Pull data until faded out. + GetAudioUntilMuted(); + // Insert new data. Timestamp is only corrected for the half of the time + // elapsed since the last packet. That is, the new packet is delayed. Verify + // that normal operation resumes. + InsertPacket(kSamples * counter_ / 2); + GetAudioUntilNormal(); +} + +// Verifies that NetEq goes out of muted state when given a future packet. +TEST_F(NetEqDecodingTestWithMutedState, MutedStateFuturePacket) { + // Insert one speech packet. + InsertPacket(0); + // Pull out audio once and expect it not to be muted. + EXPECT_FALSE(GetAudioReturnMuted()); + // Pull data until faded out. + GetAudioUntilMuted(); + // Insert new data. Timestamp is over-corrected for the time elapsed since the + // last packet. That is, the new packet is too early. Verify that normal + // operation resumes. + InsertPacket(kSamples * counter_ * 2); + GetAudioUntilNormal(); +} + +// Verifies that NetEq goes out of muted state when given an old packet. +TEST_F(NetEqDecodingTestWithMutedState, MutedStateOldPacket) { + // Insert one speech packet. + InsertPacket(0); + // Pull out audio once and expect it not to be muted. + EXPECT_FALSE(GetAudioReturnMuted()); + // Pull data until faded out. + GetAudioUntilMuted(); + + EXPECT_NE(AudioFrame::kNormalSpeech, out_frame_.speech_type_); + // Insert a few packets which are older than the first packet. + for (int i = 0; i < 5; ++i) { + InsertPacket(kSamples * (i - 1000)); + } + EXPECT_FALSE(GetAudioReturnMuted()); + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); +} + +// Verifies that NetEq doesn't enter muted state when CNG mode is active and the +// packet stream is suspended for a long time. +TEST_F(NetEqDecodingTestWithMutedState, DoNotMuteExtendedCngWithoutPackets) { + // Insert one CNG packet. + InsertCngPacket(0); + + // Pull 10 seconds of audio (10 ms audio generated per lap). + for (int i = 0; i < 1000; ++i) { + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_FALSE(muted); + } + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); +} + +// Verifies that NetEq goes back to normal after a long CNG period with the +// packet stream suspended. +TEST_F(NetEqDecodingTestWithMutedState, RecoverAfterExtendedCngWithoutPackets) { + // Insert one CNG packet. + InsertCngPacket(0); + + // Pull 10 seconds of audio (10 ms audio generated per lap). + for (int i = 0; i < 1000; ++i) { + bool muted; + EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + } + + // Insert new data. Timestamp is corrected for the time elapsed since the last + // packet. Verify that normal operation resumes. + InsertPacket(kSamples * counter_); + GetAudioUntilNormal(); +} + +namespace { +::testing::AssertionResult AudioFramesEqualExceptData(const AudioFrame& a, + const AudioFrame& b) { + if (a.timestamp_ != b.timestamp_) + return ::testing::AssertionFailure() << "timestamp_ diff (" << a.timestamp_ + << " != " << b.timestamp_ << ")"; + if (a.sample_rate_hz_ != b.sample_rate_hz_) + return ::testing::AssertionFailure() + << "sample_rate_hz_ diff (" << a.sample_rate_hz_ + << " != " << b.sample_rate_hz_ << ")"; + if (a.samples_per_channel_ != b.samples_per_channel_) + return ::testing::AssertionFailure() + << "samples_per_channel_ diff (" << a.samples_per_channel_ + << " != " << b.samples_per_channel_ << ")"; + if (a.num_channels_ != b.num_channels_) + return ::testing::AssertionFailure() + << "num_channels_ diff (" << a.num_channels_ + << " != " << b.num_channels_ << ")"; + if (a.speech_type_ != b.speech_type_) + return ::testing::AssertionFailure() + << "speech_type_ diff (" << a.speech_type_ + << " != " << b.speech_type_ << ")"; + if (a.vad_activity_ != b.vad_activity_) + return ::testing::AssertionFailure() + << "vad_activity_ diff (" << a.vad_activity_ + << " != " << b.vad_activity_ << ")"; + return ::testing::AssertionSuccess(); +} + +::testing::AssertionResult AudioFramesEqual(const AudioFrame& a, + const AudioFrame& b) { + ::testing::AssertionResult res = AudioFramesEqualExceptData(a, b); + if (!res) + return res; + if (memcmp(a.data(), b.data(), + a.samples_per_channel_ * a.num_channels_ * sizeof(*a.data())) != + 0) { + return ::testing::AssertionFailure() << "data_ diff"; + } + return ::testing::AssertionSuccess(); +} + +} // namespace + +TEST_F(NetEqDecodingTestTwoInstances, CompareMutedStateOnOff) { + ASSERT_FALSE(config_.enable_muted_state); + config2_.enable_muted_state = true; + CreateSecondInstance(); + + // Insert one speech packet into both NetEqs. + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + EXPECT_EQ(0, neteq2_->InsertPacket(rtp_info, payload)); + + AudioFrame out_frame1, out_frame2; + bool muted; + for (int i = 0; i < 1000; ++i) { + rtc::StringBuilder ss; + ss << "i = " << i; + SCOPED_TRACE(ss.str()); // Print out the loop iterator on failure. + EXPECT_EQ(0, neteq_->GetAudio(&out_frame1, &muted)); + EXPECT_FALSE(muted); + EXPECT_EQ(0, neteq2_->GetAudio(&out_frame2, &muted)); + if (muted) { + EXPECT_TRUE(AudioFramesEqualExceptData(out_frame1, out_frame2)); + } else { + EXPECT_TRUE(AudioFramesEqual(out_frame1, out_frame2)); + } + } + EXPECT_TRUE(muted); + + // Insert new data. Timestamp is corrected for the time elapsed since the last + // packet. + for (int i = 0; i < 5; ++i) { + PopulateRtpInfo(0, kSamples * 1000 + kSamples * i, &rtp_info); + EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + EXPECT_EQ(0, neteq2_->InsertPacket(rtp_info, payload)); + } + + int counter = 0; + while (out_frame1.speech_type_ != AudioFrame::kNormalSpeech) { + ASSERT_LT(counter++, 1000) << "Test timed out"; + rtc::StringBuilder ss; + ss << "counter = " << counter; + SCOPED_TRACE(ss.str()); // Print out the loop iterator on failure. + EXPECT_EQ(0, neteq_->GetAudio(&out_frame1, &muted)); + EXPECT_FALSE(muted); + EXPECT_EQ(0, neteq2_->GetAudio(&out_frame2, &muted)); + if (muted) { + EXPECT_TRUE(AudioFramesEqualExceptData(out_frame1, out_frame2)); + } else { + EXPECT_TRUE(AudioFramesEqual(out_frame1, out_frame2)); + } + } + EXPECT_FALSE(muted); +} + +TEST_F(NetEqDecodingTest, TestConcealmentEvents) { + const int kNumConcealmentEvents = 19; + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + int seq_no = 0; + RTPHeader rtp_info; + rtp_info.ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info.payloadType = 94; // PCM16b WB codec. + rtp_info.markerBit = 0; + const uint8_t payload[kPayloadBytes] = {0}; + bool muted; + + for (int i = 0; i < kNumConcealmentEvents; i++) { + // Insert some packets of 10 ms size. + for (int j = 0; j < 10; j++) { + rtp_info.sequenceNumber = seq_no++; + rtp_info.timestamp = rtp_info.sequenceNumber * kSamples; + neteq_->InsertPacket(rtp_info, payload); + neteq_->GetAudio(&out_frame_, &muted); + } + + // Lose a number of packets. + int num_lost = 1 + i; + for (int j = 0; j < num_lost; j++) { + seq_no++; + neteq_->GetAudio(&out_frame_, &muted); + } + } + + // Check number of concealment events. + NetEqLifetimeStatistics stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(kNumConcealmentEvents, static_cast(stats.concealment_events)); +} + +// Test that the jitter buffer delay stat is computed correctly. +void NetEqDecodingTestFaxMode::TestJitterBufferDelay(bool apply_packet_loss) { + const int kNumPackets = 10; + const int kDelayInNumPackets = 2; + const int kPacketLenMs = 10; // All packets are of 10 ms size. + const size_t kSamples = kPacketLenMs * 16; + const size_t kPayloadBytes = kSamples * 2; + RTPHeader rtp_info; + rtp_info.ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info.payloadType = 94; // PCM16b WB codec. + rtp_info.markerBit = 0; + const uint8_t payload[kPayloadBytes] = {0}; + bool muted; + int packets_sent = 0; + int packets_received = 0; + int expected_delay = 0; + int expected_target_delay = 0; + uint64_t expected_emitted_count = 0; + while (packets_received < kNumPackets) { + // Insert packet. + if (packets_sent < kNumPackets) { + rtp_info.sequenceNumber = packets_sent++; + rtp_info.timestamp = rtp_info.sequenceNumber * kSamples; + neteq_->InsertPacket(rtp_info, payload); + } + + // Get packet. + if (packets_sent > kDelayInNumPackets) { + neteq_->GetAudio(&out_frame_, &muted); + packets_received++; + + // The delay reported by the jitter buffer never exceeds + // the number of samples previously fetched with GetAudio + // (hence the min()). + int packets_delay = std::min(packets_received, kDelayInNumPackets + 1); + + // The increase of the expected delay is the product of + // the current delay of the jitter buffer in ms * the + // number of samples that are sent for play out. + int current_delay_ms = packets_delay * kPacketLenMs; + expected_delay += current_delay_ms * kSamples; + expected_target_delay += neteq_->TargetDelayMs() * kSamples; + expected_emitted_count += kSamples; + } + } + + if (apply_packet_loss) { + // Extra call to GetAudio to cause concealment. + neteq_->GetAudio(&out_frame_, &muted); + } + + // Check jitter buffer delay. + NetEqLifetimeStatistics stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(expected_delay, + rtc::checked_cast(stats.jitter_buffer_delay_ms)); + EXPECT_EQ(expected_emitted_count, stats.jitter_buffer_emitted_count); + EXPECT_EQ(expected_target_delay, + rtc::checked_cast(stats.jitter_buffer_target_delay_ms)); +} + +TEST_F(NetEqDecodingTestFaxMode, TestJitterBufferDelayWithoutLoss) { + TestJitterBufferDelay(false); +} + +TEST_F(NetEqDecodingTestFaxMode, TestJitterBufferDelayWithLoss) { + TestJitterBufferDelay(true); +} + +TEST_F(NetEqDecodingTestFaxMode, TestJitterBufferDelayWithAcceleration) { + const int kPacketLenMs = 10; // All packets are of 10 ms size. + const size_t kSamples = kPacketLenMs * 16; + const size_t kPayloadBytes = kSamples * 2; + RTPHeader rtp_info; + rtp_info.ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info.payloadType = 94; // PCM16b WB codec. + rtp_info.markerBit = 0; + const uint8_t payload[kPayloadBytes] = {0}; + + int expected_target_delay = neteq_->TargetDelayMs() * kSamples; + neteq_->InsertPacket(rtp_info, payload); + + bool muted; + neteq_->GetAudio(&out_frame_, &muted); + + rtp_info.sequenceNumber += 1; + rtp_info.timestamp += kSamples; + neteq_->InsertPacket(rtp_info, payload); + rtp_info.sequenceNumber += 1; + rtp_info.timestamp += kSamples; + neteq_->InsertPacket(rtp_info, payload); + + expected_target_delay += neteq_->TargetDelayMs() * 2 * kSamples; + // We have two packets in the buffer and kAccelerate operation will + // extract 20 ms of data. + neteq_->GetAudio(&out_frame_, &muted, nullptr, NetEq::Operation::kAccelerate); + + // Check jitter buffer delay. + NetEqLifetimeStatistics stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(10 * kSamples * 3, stats.jitter_buffer_delay_ms); + EXPECT_EQ(kSamples * 3, stats.jitter_buffer_emitted_count); + EXPECT_EQ(expected_target_delay, + rtc::checked_cast(stats.jitter_buffer_target_delay_ms)); +} + +namespace test { +TEST(NetEqNoTimeStretchingMode, RunTest) { + NetEq::Config config; + config.for_test_no_time_stretching = true; + auto codecs = NetEqTest::StandardDecoderMap(); + NetEqPacketSourceInput::RtpHeaderExtensionMap rtp_ext_map = { + {1, kRtpExtensionAudioLevel}, + {3, kRtpExtensionAbsoluteSendTime}, + {5, kRtpExtensionTransportSequenceNumber}, + {7, kRtpExtensionVideoContentType}, + {8, kRtpExtensionVideoTiming}}; + std::unique_ptr input(new NetEqRtpDumpInput( + webrtc::test::ResourcePath("audio_coding/neteq_universal_new", "rtp"), + rtp_ext_map, absl::nullopt /*No SSRC filter*/)); + std::unique_ptr input_time_limit( + new TimeLimitedNetEqInput(std::move(input), 20000)); + std::unique_ptr output(new VoidAudioSink); + NetEqTest::Callbacks callbacks; + NetEqTest test(config, CreateBuiltinAudioDecoderFactory(), codecs, + /*text_log=*/nullptr, /*neteq_factory=*/nullptr, + /*input=*/std::move(input_time_limit), std::move(output), + callbacks); + test.Run(); + const auto stats = test.SimulationStats(); + EXPECT_EQ(0, stats.accelerate_rate); + EXPECT_EQ(0, stats.preemptive_rate); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_unittest.proto b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_unittest.proto new file mode 100644 index 0000000000..b4b4253c3d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_unittest.proto @@ -0,0 +1,31 @@ +syntax = "proto2"; +option optimize_for = LITE_RUNTIME; +package webrtc.neteq_unittest; + +message NetEqNetworkStatistics { + // Next field number 18. + optional uint32 current_buffer_size_ms = 1; + optional uint32 preferred_buffer_size_ms = 2; + optional uint32 jitter_peaks_found = 3; + reserved 4; // Was packet_loss_rate. + reserved 5; // Was packet_discard_rate. + optional uint32 expand_rate = 6; + optional uint32 speech_expand_rate = 7; + optional uint32 preemptive_rate = 8; + optional uint32 accelerate_rate = 9; + optional uint32 secondary_decoded_rate = 10; + optional uint32 secondary_discarded_rate = 17; + optional int32 clockdrift_ppm = 11; + reserved 12; // Was added_zero_samples. + optional int32 mean_waiting_time_ms = 13; + optional int32 median_waiting_time_ms = 14; + optional int32 min_waiting_time_ms = 15; + optional int32 max_waiting_time_ms = 16; +} + +message RtcpStatistics { + optional uint32 fraction_lost = 1; + optional uint32 cumulative_lost = 2; + optional uint32 extended_max_sequence_number = 3; + optional uint32 jitter = 4; +} diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/normal.cc b/third_party/libwebrtc/modules/audio_coding/neteq/normal.cc new file mode 100644 index 0000000000..461ee7fa4a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/normal.cc @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/normal.h" + +#include // memset, memcpy + +#include // min + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/expand.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +int Normal::Process(const int16_t* input, + size_t length, + NetEq::Mode last_mode, + AudioMultiVector* output) { + if (length == 0) { + // Nothing to process. + output->Clear(); + return static_cast(length); + } + + RTC_DCHECK(output->Empty()); + // Output should be empty at this point. + if (length % output->Channels() != 0) { + // The length does not match the number of channels. + output->Clear(); + return 0; + } + output->PushBackInterleaved(rtc::ArrayView(input, length)); + + const int fs_mult = fs_hz_ / 8000; + RTC_DCHECK_GT(fs_mult, 0); + // fs_shift = log2(fs_mult), rounded down. + // Note that `fs_shift` is not "exact" for 48 kHz. + // TODO(hlundin): Investigate this further. + const int fs_shift = 30 - WebRtcSpl_NormW32(fs_mult); + + // If last call resulted in a CodedPlc we don't need to do cross-fading but we + // need to report the end of the interruption once we are back to normal + // operation. + if (last_mode == NetEq::Mode::kCodecPlc) { + statistics_->EndExpandEvent(fs_hz_); + } + + // Check if last RecOut call resulted in an Expand. If so, we have to take + // care of some cross-fading and unmuting. + if (last_mode == NetEq::Mode::kExpand) { + // Generate interpolation data using Expand. + // First, set Expand parameters to appropriate values. + expand_->SetParametersForNormalAfterExpand(); + + // Call Expand. + AudioMultiVector expanded(output->Channels()); + expand_->Process(&expanded); + expand_->Reset(); + + size_t length_per_channel = length / output->Channels(); + std::unique_ptr signal(new int16_t[length_per_channel]); + for (size_t channel_ix = 0; channel_ix < output->Channels(); ++channel_ix) { + // Set muting factor to the same as expand muting factor. + int16_t mute_factor = expand_->MuteFactor(channel_ix); + + (*output)[channel_ix].CopyTo(length_per_channel, 0, signal.get()); + + // Find largest absolute value in new data. + int16_t decoded_max = + WebRtcSpl_MaxAbsValueW16(signal.get(), length_per_channel); + // Adjust muting factor if needed (to BGN level). + size_t energy_length = + std::min(static_cast(fs_mult * 64), length_per_channel); + int scaling = 6 + fs_shift - WebRtcSpl_NormW32(decoded_max * decoded_max); + scaling = std::max(scaling, 0); // `scaling` should always be >= 0. + int32_t energy = WebRtcSpl_DotProductWithScale(signal.get(), signal.get(), + energy_length, scaling); + int32_t scaled_energy_length = + static_cast(energy_length >> scaling); + if (scaled_energy_length > 0) { + energy = energy / scaled_energy_length; + } else { + energy = 0; + } + + int local_mute_factor = 16384; // 1.0 in Q14. + if ((energy != 0) && (energy > background_noise_.Energy(channel_ix))) { + // Normalize new frame energy to 15 bits. + scaling = WebRtcSpl_NormW32(energy) - 16; + // We want background_noise_.energy() / energy in Q14. + int32_t bgn_energy = WEBRTC_SPL_SHIFT_W32( + background_noise_.Energy(channel_ix), scaling + 14); + int16_t energy_scaled = + static_cast(WEBRTC_SPL_SHIFT_W32(energy, scaling)); + int32_t ratio = WebRtcSpl_DivW32W16(bgn_energy, energy_scaled); + local_mute_factor = + std::min(local_mute_factor, WebRtcSpl_SqrtFloor(ratio << 14)); + } + mute_factor = std::max(mute_factor, local_mute_factor); + RTC_DCHECK_LE(mute_factor, 16384); + RTC_DCHECK_GE(mute_factor, 0); + + // If muted increase by 0.64 for every 20 ms (NB/WB 0.0040/0.0020 in Q14), + // or as fast as it takes to come back to full gain within the frame + // length. + const int back_to_fullscale_inc = + static_cast((16384 - mute_factor) / length_per_channel); + const int increment = std::max(64 / fs_mult, back_to_fullscale_inc); + for (size_t i = 0; i < length_per_channel; i++) { + // Scale with mute factor. + RTC_DCHECK_LT(channel_ix, output->Channels()); + RTC_DCHECK_LT(i, output->Size()); + int32_t scaled_signal = (*output)[channel_ix][i] * mute_factor; + // Shift 14 with proper rounding. + (*output)[channel_ix][i] = + static_cast((scaled_signal + 8192) >> 14); + // Increase mute_factor towards 16384. + mute_factor = + static_cast(std::min(mute_factor + increment, 16384)); + } + + // Interpolate the expanded data into the new vector. + // (NB/WB/SWB32/SWB48 8/16/32/48 samples.) + size_t win_length = samples_per_ms_; + int16_t win_slope_Q14 = default_win_slope_Q14_; + RTC_DCHECK_LT(channel_ix, output->Channels()); + if (win_length > output->Size()) { + win_length = output->Size(); + win_slope_Q14 = (1 << 14) / static_cast(win_length); + } + int16_t win_up_Q14 = 0; + for (size_t i = 0; i < win_length; i++) { + win_up_Q14 += win_slope_Q14; + (*output)[channel_ix][i] = + (win_up_Q14 * (*output)[channel_ix][i] + + ((1 << 14) - win_up_Q14) * expanded[channel_ix][i] + (1 << 13)) >> + 14; + } + RTC_DCHECK_GT(win_up_Q14, + (1 << 14) - 32); // Worst case rouding is a length of 34 + } + } else if (last_mode == NetEq::Mode::kRfc3389Cng) { + RTC_DCHECK_EQ(output->Channels(), 1); // Not adapted for multi-channel yet. + static const size_t kCngLength = 48; + RTC_DCHECK_LE(8 * fs_mult, kCngLength); + int16_t cng_output[kCngLength]; + ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder(); + + if (cng_decoder) { + // Generate long enough for 48kHz. + if (!cng_decoder->Generate(cng_output, false)) { + // Error returned; set return vector to all zeros. + memset(cng_output, 0, sizeof(cng_output)); + } + } else { + // If no CNG instance is defined, just copy from the decoded data. + // (This will result in interpolating the decoded with itself.) + (*output)[0].CopyTo(fs_mult * 8, 0, cng_output); + } + // Interpolate the CNG into the new vector. + // (NB/WB/SWB32/SWB48 8/16/32/48 samples.) + size_t win_length = samples_per_ms_; + int16_t win_slope_Q14 = default_win_slope_Q14_; + if (win_length > kCngLength) { + win_length = kCngLength; + win_slope_Q14 = (1 << 14) / static_cast(win_length); + } + int16_t win_up_Q14 = 0; + for (size_t i = 0; i < win_length; i++) { + win_up_Q14 += win_slope_Q14; + (*output)[0][i] = + (win_up_Q14 * (*output)[0][i] + + ((1 << 14) - win_up_Q14) * cng_output[i] + (1 << 13)) >> + 14; + } + RTC_DCHECK_GT(win_up_Q14, + (1 << 14) - 32); // Worst case rouding is a length of 34 + } + + return static_cast(length); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/normal.h b/third_party/libwebrtc/modules/audio_coding/neteq/normal.h new file mode 100644 index 0000000000..772293b605 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/normal.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_NORMAL_H_ +#define MODULES_AUDIO_CODING_NETEQ_NORMAL_H_ + +#include +#include // Access to size_t. + +#include "api/neteq/neteq.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +// Forward declarations. +class AudioMultiVector; +class BackgroundNoise; +class DecoderDatabase; +class Expand; + +// This class provides the "Normal" DSP operation, that is performed when +// there is no data loss, no need to stretch the timing of the signal, and +// no other "special circumstances" are at hand. +class Normal { + public: + Normal(int fs_hz, + DecoderDatabase* decoder_database, + const BackgroundNoise& background_noise, + Expand* expand, + StatisticsCalculator* statistics) + : fs_hz_(fs_hz), + decoder_database_(decoder_database), + background_noise_(background_noise), + expand_(expand), + samples_per_ms_(rtc::CheckedDivExact(fs_hz_, 1000)), + default_win_slope_Q14_( + rtc::dchecked_cast((1 << 14) / samples_per_ms_)), + statistics_(statistics) {} + + virtual ~Normal() {} + + Normal(const Normal&) = delete; + Normal& operator=(const Normal&) = delete; + + // Performs the "Normal" operation. The decoder data is supplied in `input`, + // having `length` samples in total for all channels (interleaved). The + // result is written to `output`. The number of channels allocated in + // `output` defines the number of channels that will be used when + // de-interleaving `input`. `last_mode` contains the mode used in the previous + // GetAudio call (i.e., not the current one). + int Process(const int16_t* input, + size_t length, + NetEq::Mode last_mode, + AudioMultiVector* output); + + private: + int fs_hz_; + DecoderDatabase* decoder_database_; + const BackgroundNoise& background_noise_; + Expand* expand_; + const size_t samples_per_ms_; + const int16_t default_win_slope_Q14_; + StatisticsCalculator* const statistics_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_NORMAL_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/normal_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/normal_unittest.cc new file mode 100644 index 0000000000..4554d79576 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/normal_unittest.cc @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for Normal class. + +#include "modules/audio_coding/neteq/normal.h" + +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/expand.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/mock/mock_expand.h" +#include "modules/audio_coding/neteq/random_vector.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "modules/audio_coding/neteq/sync_buffer.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::Invoke; + +namespace webrtc { + +namespace { + +int ExpandProcess120ms(AudioMultiVector* output) { + AudioMultiVector dummy_audio(1, 11520u); + dummy_audio.CopyTo(output); + return 0; +} + +} // namespace + +TEST(Normal, CreateAndDestroy) { + MockDecoderDatabase db; + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); + Normal normal(fs, &db, bgn, &expand, &statistics); + EXPECT_CALL(db, Die()); // Called when `db` goes out of scope. +} + +TEST(Normal, AvoidDivideByZero) { + MockDecoderDatabase db; + int fs = 8000; + size_t channels = 1; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(1, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, + channels); + Normal normal(fs, &db, bgn, &expand, &statistics); + + int16_t input[1000] = {0}; + AudioMultiVector output(channels); + + // Zero input length. + EXPECT_EQ(0, normal.Process(input, 0, NetEq::Mode::kExpand, &output)); + EXPECT_EQ(0u, output.Size()); + + // Try to make energy_length >> scaling = 0; + EXPECT_CALL(expand, SetParametersForNormalAfterExpand()); + EXPECT_CALL(expand, Process(_)); + EXPECT_CALL(expand, Reset()); + // If input_size_samples < 64, then energy_length in Normal::Process() will + // be equal to input_size_samples. Since the input is all zeros, decoded_max + // will be zero, and scaling will be >= 6. Thus, energy_length >> scaling = 0, + // and using this as a denominator would lead to problems. + int input_size_samples = 63; + EXPECT_EQ(input_size_samples, normal.Process(input, input_size_samples, + NetEq::Mode::kExpand, &output)); + + EXPECT_CALL(db, Die()); // Called when `db` goes out of scope. + EXPECT_CALL(expand, Die()); // Called when `expand` goes out of scope. +} + +TEST(Normal, InputLengthAndChannelsDoNotMatch) { + MockDecoderDatabase db; + int fs = 8000; + size_t channels = 2; + BackgroundNoise bgn(channels); + SyncBuffer sync_buffer(channels, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, + channels); + Normal normal(fs, &db, bgn, &expand, &statistics); + + int16_t input[1000] = {0}; + AudioMultiVector output(channels); + + // Let the number of samples be one sample less than 80 samples per channel. + size_t input_len = 80 * channels - 1; + EXPECT_EQ(0, normal.Process(input, input_len, NetEq::Mode::kExpand, &output)); + EXPECT_EQ(0u, output.Size()); + + EXPECT_CALL(db, Die()); // Called when `db` goes out of scope. + EXPECT_CALL(expand, Die()); // Called when `expand` goes out of scope. +} + +TEST(Normal, LastModeExpand120msPacket) { + MockDecoderDatabase db; + const int kFs = 48000; + const size_t kPacketsizeBytes = 11520u; + const size_t kChannels = 1; + BackgroundNoise bgn(kChannels); + SyncBuffer sync_buffer(kChannels, 1000); + RandomVector random_vector; + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, kFs, + kChannels); + Normal normal(kFs, &db, bgn, &expand, &statistics); + + int16_t input[kPacketsizeBytes] = {0}; + AudioMultiVector output(kChannels); + + EXPECT_CALL(expand, SetParametersForNormalAfterExpand()); + EXPECT_CALL(expand, Process(_)).WillOnce(Invoke(ExpandProcess120ms)); + EXPECT_CALL(expand, Reset()); + EXPECT_EQ( + static_cast(kPacketsizeBytes), + normal.Process(input, kPacketsizeBytes, NetEq::Mode::kExpand, &output)); + + EXPECT_EQ(kPacketsizeBytes, output.Size()); + + EXPECT_CALL(db, Die()); // Called when `db` goes out of scope. + EXPECT_CALL(expand, Die()); // Called when `expand` goes out of scope. +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/packet.cc b/third_party/libwebrtc/modules/audio_coding/neteq/packet.cc new file mode 100644 index 0000000000..333f161229 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/packet.cc @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/packet.h" + +namespace webrtc { + +Packet::Packet() = default; +Packet::Packet(Packet&& b) = default; + +Packet::~Packet() = default; + +Packet& Packet::operator=(Packet&& b) = default; + +Packet Packet::Clone() const { + RTC_CHECK(!frame); + + Packet clone; + clone.timestamp = timestamp; + clone.sequence_number = sequence_number; + clone.payload_type = payload_type; + clone.payload.SetData(payload.data(), payload.size()); + clone.priority = priority; + clone.packet_info = packet_info; + + return clone; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/packet.h b/third_party/libwebrtc/modules/audio_coding/neteq/packet.h new file mode 100644 index 0000000000..0c6f204edb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/packet.h @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_PACKET_H_ +#define MODULES_AUDIO_CODING_NETEQ_PACKET_H_ + +#include + +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "api/neteq/tick_timer.h" +#include "api/rtp_packet_info.h" +#include "rtc_base/buffer.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// Struct for holding RTP packets. +struct Packet { + struct Priority { + Priority() : codec_level(0), red_level(0) {} + Priority(int codec_level, int red_level) + : codec_level(codec_level), red_level(red_level) { + CheckInvariant(); + } + + int codec_level; + int red_level; + + // Priorities are sorted low-to-high, first on the level the codec + // prioritizes it, then on the level of RED packet it is; i.e. if it is a + // primary or secondary payload of a RED packet. For example: with Opus, an + // Fec packet (which the decoder prioritizes lower than a regular packet) + // will not be used if there is _any_ RED payload for the same + // timeframe. The highest priority packet will have levels {0, 0}. Negative + // priorities are not allowed. + bool operator<(const Priority& b) const { + CheckInvariant(); + b.CheckInvariant(); + if (codec_level == b.codec_level) + return red_level < b.red_level; + + return codec_level < b.codec_level; + } + bool operator==(const Priority& b) const { + CheckInvariant(); + b.CheckInvariant(); + return codec_level == b.codec_level && red_level == b.red_level; + } + bool operator!=(const Priority& b) const { return !(*this == b); } + bool operator>(const Priority& b) const { return b < *this; } + bool operator<=(const Priority& b) const { return !(b > *this); } + bool operator>=(const Priority& b) const { return !(b < *this); } + + private: + void CheckInvariant() const { + RTC_DCHECK_GE(codec_level, 0); + RTC_DCHECK_GE(red_level, 0); + } + }; + + uint32_t timestamp; + uint16_t sequence_number; + uint8_t payload_type; + // Datagram excluding RTP header and header extension. + rtc::Buffer payload; + Priority priority; + RtpPacketInfo packet_info; + std::unique_ptr waiting_time; + std::unique_ptr frame; + + Packet(); + Packet(Packet&& b); + ~Packet(); + + // Packets should generally be moved around but sometimes it's useful to make + // a copy, for example for testing purposes. NOTE: Will only work for + // un-parsed packets, i.e. `frame` must be unset. The payload will, however, + // be copied. `waiting_time` will also not be copied. + Packet Clone() const; + + Packet& operator=(Packet&& b); + + // Comparison operators. Establish a packet ordering based on (1) timestamp, + // (2) sequence number and (3) redundancy. + // Timestamp and sequence numbers are compared taking wrap-around into + // account. For two packets with the same sequence number and timestamp a + // primary payload is considered "smaller" than a secondary. + bool operator==(const Packet& rhs) const { + return (this->timestamp == rhs.timestamp && + this->sequence_number == rhs.sequence_number && + this->priority == rhs.priority); + } + bool operator!=(const Packet& rhs) const { return !operator==(rhs); } + bool operator<(const Packet& rhs) const { + if (this->timestamp == rhs.timestamp) { + if (this->sequence_number == rhs.sequence_number) { + // Timestamp and sequence numbers are identical - deem the left hand + // side to be "smaller" (i.e., "earlier") if it has higher priority. + return this->priority < rhs.priority; + } + return (static_cast(rhs.sequence_number - + this->sequence_number) < 0xFFFF / 2); + } + return (static_cast(rhs.timestamp - this->timestamp) < + 0xFFFFFFFF / 2); + } + bool operator>(const Packet& rhs) const { return rhs.operator<(*this); } + bool operator<=(const Packet& rhs) const { return !operator>(rhs); } + bool operator>=(const Packet& rhs) const { return !operator<(rhs); } + + bool empty() const { return !frame && payload.empty(); } +}; + +// A list of packets. +typedef std::list PacketList; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_PACKET_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.cc b/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.cc new file mode 100644 index 0000000000..2077383f76 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/packet_arrival_history.h" + +#include + +#include "api/neteq/tick_timer.h" + +namespace webrtc { + +PacketArrivalHistory::PacketArrivalHistory(int window_size_ms) + : window_size_ms_(window_size_ms) {} + +void PacketArrivalHistory::Insert(uint32_t rtp_timestamp, + int64_t arrival_time_ms) { + RTC_DCHECK(sample_rate_khz_ > 0); + int64_t unwrapped_rtp_timestamp = timestamp_unwrapper_.Unwrap(rtp_timestamp); + if (!newest_rtp_timestamp_ || + unwrapped_rtp_timestamp > *newest_rtp_timestamp_) { + newest_rtp_timestamp_ = unwrapped_rtp_timestamp; + } + history_.emplace_back(unwrapped_rtp_timestamp / sample_rate_khz_, + arrival_time_ms); + MaybeUpdateCachedArrivals(history_.back()); + while (history_.front().rtp_timestamp_ms + window_size_ms_ < + unwrapped_rtp_timestamp / sample_rate_khz_) { + if (&history_.front() == min_packet_arrival_) { + min_packet_arrival_ = nullptr; + } + if (&history_.front() == max_packet_arrival_) { + max_packet_arrival_ = nullptr; + } + history_.pop_front(); + } + if (!min_packet_arrival_ || !max_packet_arrival_) { + for (const PacketArrival& packet : history_) { + MaybeUpdateCachedArrivals(packet); + } + } +} + +void PacketArrivalHistory::MaybeUpdateCachedArrivals( + const PacketArrival& packet_arrival) { + if (!min_packet_arrival_ || packet_arrival <= *min_packet_arrival_) { + min_packet_arrival_ = &packet_arrival; + } + if (!max_packet_arrival_ || packet_arrival >= *max_packet_arrival_) { + max_packet_arrival_ = &packet_arrival; + } +} + +void PacketArrivalHistory::Reset() { + history_.clear(); + min_packet_arrival_ = nullptr; + max_packet_arrival_ = nullptr; + timestamp_unwrapper_.Reset(); + newest_rtp_timestamp_ = absl::nullopt; +} + +int PacketArrivalHistory::GetDelayMs(uint32_t rtp_timestamp, + int64_t time_ms) const { + RTC_DCHECK(sample_rate_khz_ > 0); + int64_t unwrapped_rtp_timestamp_ms = + timestamp_unwrapper_.PeekUnwrap(rtp_timestamp) / sample_rate_khz_; + PacketArrival packet(unwrapped_rtp_timestamp_ms, time_ms); + return GetPacketArrivalDelayMs(packet); +} + +int PacketArrivalHistory::GetMaxDelayMs() const { + if (!max_packet_arrival_) { + return 0; + } + return GetPacketArrivalDelayMs(*max_packet_arrival_); +} + +bool PacketArrivalHistory::IsNewestRtpTimestamp(uint32_t rtp_timestamp) const { + if (!newest_rtp_timestamp_) { + return false; + } + int64_t unwrapped_rtp_timestamp = + timestamp_unwrapper_.PeekUnwrap(rtp_timestamp); + return unwrapped_rtp_timestamp == *newest_rtp_timestamp_; +} + +int PacketArrivalHistory::GetPacketArrivalDelayMs( + const PacketArrival& packet_arrival) const { + if (!min_packet_arrival_) { + return 0; + } + return std::max(static_cast(packet_arrival.arrival_time_ms - + min_packet_arrival_->arrival_time_ms - + (packet_arrival.rtp_timestamp_ms - + min_packet_arrival_->rtp_timestamp_ms)), + 0); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.h b/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.h new file mode 100644 index 0000000000..cad362b469 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_PACKET_ARRIVAL_HISTORY_H_ +#define MODULES_AUDIO_CODING_NETEQ_PACKET_ARRIVAL_HISTORY_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/neteq/tick_timer.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" + +namespace webrtc { + +// Stores timing information about previously received packets. +// The history has a fixed window size beyond which old data is automatically +// pruned. +class PacketArrivalHistory { + public: + explicit PacketArrivalHistory(int window_size_ms); + + // Insert packet with `rtp_timestamp` and `arrival_time_ms` into the history. + void Insert(uint32_t rtp_timestamp, int64_t arrival_time_ms); + + // The delay for `rtp_timestamp` at `time_ms` is calculated as + // `(time_ms - p.arrival_time_ms) - (rtp_timestamp - p.rtp_timestamp)` + // where `p` is chosen as the packet arrival in the history that maximizes the + // delay. + int GetDelayMs(uint32_t rtp_timestamp, int64_t time_ms) const; + + // Get the maximum packet arrival delay observed in the history. + int GetMaxDelayMs() const; + + bool IsNewestRtpTimestamp(uint32_t rtp_timestamp) const; + + void Reset(); + + void set_sample_rate(int sample_rate) { + sample_rate_khz_ = sample_rate / 1000; + } + + size_t size() const { return history_.size(); } + + private: + struct PacketArrival { + PacketArrival(int64_t rtp_timestamp_ms, int64_t arrival_time_ms) + : rtp_timestamp_ms(rtp_timestamp_ms), + arrival_time_ms(arrival_time_ms) {} + int64_t rtp_timestamp_ms; + int64_t arrival_time_ms; + bool operator<=(const PacketArrival& other) const { + return arrival_time_ms - rtp_timestamp_ms <= + other.arrival_time_ms - other.rtp_timestamp_ms; + } + bool operator>=(const PacketArrival& other) const { + return arrival_time_ms - rtp_timestamp_ms >= + other.arrival_time_ms - other.rtp_timestamp_ms; + } + }; + std::deque history_; + int GetPacketArrivalDelayMs(const PacketArrival& packet_arrival) const; + // Updates `min_packet_arrival_` and `max_packet_arrival_`. + void MaybeUpdateCachedArrivals(const PacketArrival& packet); + const PacketArrival* min_packet_arrival_ = nullptr; + const PacketArrival* max_packet_arrival_ = nullptr; + const int window_size_ms_; + RtpTimestampUnwrapper timestamp_unwrapper_; + absl::optional newest_rtp_timestamp_; + int sample_rate_khz_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_PACKET_ARRIVAL_HISTORY_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history_unittest.cc new file mode 100644 index 0000000000..539a318fe1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history_unittest.cc @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/packet_arrival_history.h" + +#include +#include + +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr int kFs = 8000; +constexpr int kFsKhz = kFs / 1000; +constexpr int kFrameSizeMs = 20; +constexpr int kWindowSizeMs = 1000; + +class PacketArrivalHistoryTest : public testing::Test { + public: + PacketArrivalHistoryTest() : history_(kWindowSizeMs) { + history_.set_sample_rate(kFs); + } + void IncrementTime(int delta_ms) { time_ms_ += delta_ms; } + int InsertPacketAndGetDelay(int timestamp_delta_ms) { + uint32_t timestamp = timestamp_ + timestamp_delta_ms * kFsKhz; + if (timestamp_delta_ms > 0) { + timestamp_ = timestamp; + } + history_.Insert(timestamp, time_ms_); + EXPECT_EQ(history_.IsNewestRtpTimestamp(timestamp), + timestamp_delta_ms >= 0); + return history_.GetDelayMs(timestamp, time_ms_); + } + + protected: + int64_t time_ms_ = 0; + PacketArrivalHistory history_; + uint32_t timestamp_ = 0x12345678; +}; + +TEST_F(PacketArrivalHistoryTest, RelativeArrivalDelay) { + EXPECT_EQ(InsertPacketAndGetDelay(0), 0); + + IncrementTime(kFrameSizeMs); + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), 0); + + IncrementTime(2 * kFrameSizeMs); + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), 20); + + // Reordered packet. + EXPECT_EQ(InsertPacketAndGetDelay(-2 * kFrameSizeMs), 60); + + IncrementTime(2 * kFrameSizeMs); + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), 40); + + // Move reference packet forward. + EXPECT_EQ(InsertPacketAndGetDelay(4 * kFrameSizeMs), 0); + + IncrementTime(2 * kFrameSizeMs); + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), 20); + + // Earlier packet is now more delayed due to the new reference packet. + EXPECT_EQ(history_.GetMaxDelayMs(), 100); +} + +TEST_F(PacketArrivalHistoryTest, ReorderedPackets) { + // Insert first packet. + EXPECT_EQ(InsertPacketAndGetDelay(0), 0); + + // Insert reordered packet. + EXPECT_EQ(InsertPacketAndGetDelay(-80), 80); + + // Insert another reordered packet. + EXPECT_EQ(InsertPacketAndGetDelay(-kFrameSizeMs), 20); + + // Insert the next packet in order and verify that the relative delay is + // estimated based on the first inserted packet. + IncrementTime(4 * kFrameSizeMs); + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), 60); + + EXPECT_EQ(history_.GetMaxDelayMs(), 80); +} + +TEST_F(PacketArrivalHistoryTest, MaxHistorySize) { + EXPECT_EQ(InsertPacketAndGetDelay(0), 0); + + IncrementTime(2 * kFrameSizeMs); + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), 20); + EXPECT_EQ(history_.GetMaxDelayMs(), 20); + + // Insert next packet with a timestamp difference larger than maximum history + // size. This removes the previously inserted packet from the history. + IncrementTime(kWindowSizeMs + kFrameSizeMs); + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs + kWindowSizeMs), 0); + EXPECT_EQ(history_.GetMaxDelayMs(), 0); +} + +TEST_F(PacketArrivalHistoryTest, TimestampWraparound) { + timestamp_ = std::numeric_limits::max(); + EXPECT_EQ(InsertPacketAndGetDelay(0), 0); + + IncrementTime(2 * kFrameSizeMs); + // Insert timestamp that will wrap around. + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), kFrameSizeMs); + + // Insert reordered packet before the wraparound. + EXPECT_EQ(InsertPacketAndGetDelay(-2 * kFrameSizeMs), 3 * kFrameSizeMs); + + // Insert another in-order packet after the wraparound. + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), 0); + + EXPECT_EQ(history_.GetMaxDelayMs(), 3 * kFrameSizeMs); +} + +TEST_F(PacketArrivalHistoryTest, TimestampWraparoundBackwards) { + timestamp_ = 0; + EXPECT_EQ(InsertPacketAndGetDelay(0), 0); + + IncrementTime(2 * kFrameSizeMs); + // Insert timestamp that will wrap around. + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), kFrameSizeMs); + + // Insert reordered packet before the wraparound. + EXPECT_EQ(InsertPacketAndGetDelay(-2 * kFrameSizeMs), 3 * kFrameSizeMs); + + // Insert another in-order packet after the wraparound. + EXPECT_EQ(InsertPacketAndGetDelay(kFrameSizeMs), 0); + + EXPECT_EQ(history_.GetMaxDelayMs(), 3 * kFrameSizeMs); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.cc b/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.cc new file mode 100644 index 0000000000..f6b5a476c9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.cc @@ -0,0 +1,405 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This is the implementation of the PacketBuffer class. It is mostly based on +// an STL list. The list is kept sorted at all times so that the next packet to +// decode is at the beginning of the list. + +#include "modules/audio_coding/neteq/packet_buffer.h" + +#include +#include +#include +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "api/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/statistics_calculator.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/struct_parameters_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace { +// Predicate used when inserting packets in the buffer list. +// Operator() returns true when `packet` goes before `new_packet`. +class NewTimestampIsLarger { + public: + explicit NewTimestampIsLarger(const Packet& new_packet) + : new_packet_(new_packet) {} + bool operator()(const Packet& packet) { return (new_packet_ >= packet); } + + private: + const Packet& new_packet_; +}; + +// Returns true if both payload types are known to the decoder database, and +// have the same sample rate. +bool EqualSampleRates(uint8_t pt1, + uint8_t pt2, + const DecoderDatabase& decoder_database) { + auto* di1 = decoder_database.GetDecoderInfo(pt1); + auto* di2 = decoder_database.GetDecoderInfo(pt2); + return di1 && di2 && di1->SampleRateHz() == di2->SampleRateHz(); +} + +void LogPacketDiscarded(int codec_level, StatisticsCalculator* stats) { + RTC_CHECK(stats); + if (codec_level > 0) { + stats->SecondaryPacketsDiscarded(1); + } else { + stats->PacketsDiscarded(1); + } +} + +absl::optional GetSmartflushingConfig() { + absl::optional result; + std::string field_trial_string = + field_trial::FindFullName("WebRTC-Audio-NetEqSmartFlushing"); + result = SmartFlushingConfig(); + bool enabled = false; + auto parser = StructParametersParser::Create( + "enabled", &enabled, "target_level_threshold_ms", + &result->target_level_threshold_ms, "target_level_multiplier", + &result->target_level_multiplier); + parser->Parse(field_trial_string); + if (!enabled) { + return absl::nullopt; + } + RTC_LOG(LS_INFO) << "Using smart flushing, target_level_threshold_ms: " + << result->target_level_threshold_ms + << ", target_level_multiplier: " + << result->target_level_multiplier; + return result; +} + +} // namespace + +PacketBuffer::PacketBuffer(size_t max_number_of_packets, + const TickTimer* tick_timer) + : smart_flushing_config_(GetSmartflushingConfig()), + max_number_of_packets_(max_number_of_packets), + tick_timer_(tick_timer) {} + +// Destructor. All packets in the buffer will be destroyed. +PacketBuffer::~PacketBuffer() { + buffer_.clear(); +} + +// Flush the buffer. All packets in the buffer will be destroyed. +void PacketBuffer::Flush(StatisticsCalculator* stats) { + for (auto& p : buffer_) { + LogPacketDiscarded(p.priority.codec_level, stats); + } + buffer_.clear(); + stats->FlushedPacketBuffer(); +} + +void PacketBuffer::PartialFlush(int target_level_ms, + size_t sample_rate, + size_t last_decoded_length, + StatisticsCalculator* stats) { + // Make sure that at least half the packet buffer capacity will be available + // after the flush. This is done to avoid getting stuck if the target level is + // very high. + int target_level_samples = + std::min(target_level_ms * sample_rate / 1000, + max_number_of_packets_ * last_decoded_length / 2); + // We should avoid flushing to very low levels. + target_level_samples = std::max( + target_level_samples, smart_flushing_config_->target_level_threshold_ms); + while (GetSpanSamples(last_decoded_length, sample_rate, true) > + static_cast(target_level_samples) || + buffer_.size() > max_number_of_packets_ / 2) { + LogPacketDiscarded(PeekNextPacket()->priority.codec_level, stats); + buffer_.pop_front(); + } +} + +bool PacketBuffer::Empty() const { + return buffer_.empty(); +} + +int PacketBuffer::InsertPacket(Packet&& packet, + StatisticsCalculator* stats, + size_t last_decoded_length, + size_t sample_rate, + int target_level_ms, + const DecoderDatabase& decoder_database) { + if (packet.empty()) { + RTC_LOG(LS_WARNING) << "InsertPacket invalid packet"; + return kInvalidPacket; + } + + RTC_DCHECK_GE(packet.priority.codec_level, 0); + RTC_DCHECK_GE(packet.priority.red_level, 0); + + int return_val = kOK; + + packet.waiting_time = tick_timer_->GetNewStopwatch(); + + // Perform a smart flush if the buffer size exceeds a multiple of the target + // level. + const size_t span_threshold = + smart_flushing_config_ + ? smart_flushing_config_->target_level_multiplier * + std::max(smart_flushing_config_->target_level_threshold_ms, + target_level_ms) * + sample_rate / 1000 + : 0; + const bool smart_flush = + smart_flushing_config_.has_value() && + GetSpanSamples(last_decoded_length, sample_rate, true) >= span_threshold; + if (buffer_.size() >= max_number_of_packets_ || smart_flush) { + size_t buffer_size_before_flush = buffer_.size(); + if (smart_flushing_config_.has_value()) { + // Flush down to the target level. + PartialFlush(target_level_ms, sample_rate, last_decoded_length, stats); + return_val = kPartialFlush; + } else { + // Buffer is full. + Flush(stats); + return_val = kFlushed; + } + RTC_LOG(LS_WARNING) << "Packet buffer flushed, " + << (buffer_size_before_flush - buffer_.size()) + << " packets discarded."; + } + + // Get an iterator pointing to the place in the buffer where the new packet + // should be inserted. The list is searched from the back, since the most + // likely case is that the new packet should be near the end of the list. + PacketList::reverse_iterator rit = std::find_if( + buffer_.rbegin(), buffer_.rend(), NewTimestampIsLarger(packet)); + + // The new packet is to be inserted to the right of `rit`. If it has the same + // timestamp as `rit`, which has a higher priority, do not insert the new + // packet to list. + if (rit != buffer_.rend() && packet.timestamp == rit->timestamp) { + LogPacketDiscarded(packet.priority.codec_level, stats); + return return_val; + } + + // The new packet is to be inserted to the left of `it`. If it has the same + // timestamp as `it`, which has a lower priority, replace `it` with the new + // packet. + PacketList::iterator it = rit.base(); + if (it != buffer_.end() && packet.timestamp == it->timestamp) { + LogPacketDiscarded(it->priority.codec_level, stats); + it = buffer_.erase(it); + } + buffer_.insert(it, std::move(packet)); // Insert the packet at that position. + + return return_val; +} + +int PacketBuffer::InsertPacketList( + PacketList* packet_list, + const DecoderDatabase& decoder_database, + absl::optional* current_rtp_payload_type, + absl::optional* current_cng_rtp_payload_type, + StatisticsCalculator* stats, + size_t last_decoded_length, + size_t sample_rate, + int target_level_ms) { + RTC_DCHECK(stats); + bool flushed = false; + for (auto& packet : *packet_list) { + if (decoder_database.IsComfortNoise(packet.payload_type)) { + if (*current_cng_rtp_payload_type && + **current_cng_rtp_payload_type != packet.payload_type) { + // New CNG payload type implies new codec type. + *current_rtp_payload_type = absl::nullopt; + Flush(stats); + flushed = true; + } + *current_cng_rtp_payload_type = packet.payload_type; + } else if (!decoder_database.IsDtmf(packet.payload_type)) { + // This must be speech. + if ((*current_rtp_payload_type && + **current_rtp_payload_type != packet.payload_type) || + (*current_cng_rtp_payload_type && + !EqualSampleRates(packet.payload_type, + **current_cng_rtp_payload_type, + decoder_database))) { + *current_cng_rtp_payload_type = absl::nullopt; + Flush(stats); + flushed = true; + } + *current_rtp_payload_type = packet.payload_type; + } + int return_val = + InsertPacket(std::move(packet), stats, last_decoded_length, sample_rate, + target_level_ms, decoder_database); + if (return_val == kFlushed) { + // The buffer flushed, but this is not an error. We can still continue. + flushed = true; + } else if (return_val != kOK) { + // An error occurred. Delete remaining packets in list and return. + packet_list->clear(); + return return_val; + } + } + packet_list->clear(); + return flushed ? kFlushed : kOK; +} + +int PacketBuffer::NextTimestamp(uint32_t* next_timestamp) const { + if (Empty()) { + return kBufferEmpty; + } + if (!next_timestamp) { + return kInvalidPointer; + } + *next_timestamp = buffer_.front().timestamp; + return kOK; +} + +int PacketBuffer::NextHigherTimestamp(uint32_t timestamp, + uint32_t* next_timestamp) const { + if (Empty()) { + return kBufferEmpty; + } + if (!next_timestamp) { + return kInvalidPointer; + } + PacketList::const_iterator it; + for (it = buffer_.begin(); it != buffer_.end(); ++it) { + if (it->timestamp >= timestamp) { + // Found a packet matching the search. + *next_timestamp = it->timestamp; + return kOK; + } + } + return kNotFound; +} + +const Packet* PacketBuffer::PeekNextPacket() const { + return buffer_.empty() ? nullptr : &buffer_.front(); +} + +absl::optional PacketBuffer::GetNextPacket() { + if (Empty()) { + // Buffer is empty. + return absl::nullopt; + } + + absl::optional packet(std::move(buffer_.front())); + // Assert that the packet sanity checks in InsertPacket method works. + RTC_DCHECK(!packet->empty()); + buffer_.pop_front(); + + return packet; +} + +int PacketBuffer::DiscardNextPacket(StatisticsCalculator* stats) { + if (Empty()) { + return kBufferEmpty; + } + // Assert that the packet sanity checks in InsertPacket method works. + const Packet& packet = buffer_.front(); + RTC_DCHECK(!packet.empty()); + LogPacketDiscarded(packet.priority.codec_level, stats); + buffer_.pop_front(); + return kOK; +} + +void PacketBuffer::DiscardOldPackets(uint32_t timestamp_limit, + uint32_t horizon_samples, + StatisticsCalculator* stats) { + buffer_.remove_if([timestamp_limit, horizon_samples, stats](const Packet& p) { + if (timestamp_limit == p.timestamp || + !IsObsoleteTimestamp(p.timestamp, timestamp_limit, horizon_samples)) { + return false; + } + LogPacketDiscarded(p.priority.codec_level, stats); + return true; + }); +} + +void PacketBuffer::DiscardAllOldPackets(uint32_t timestamp_limit, + StatisticsCalculator* stats) { + DiscardOldPackets(timestamp_limit, 0, stats); +} + +void PacketBuffer::DiscardPacketsWithPayloadType(uint8_t payload_type, + StatisticsCalculator* stats) { + buffer_.remove_if([payload_type, stats](const Packet& p) { + if (p.payload_type != payload_type) { + return false; + } + LogPacketDiscarded(p.priority.codec_level, stats); + return true; + }); +} + +size_t PacketBuffer::NumPacketsInBuffer() const { + return buffer_.size(); +} + +size_t PacketBuffer::NumSamplesInBuffer(size_t last_decoded_length) const { + size_t num_samples = 0; + size_t last_duration = last_decoded_length; + for (const Packet& packet : buffer_) { + if (packet.frame) { + // TODO(hlundin): Verify that it's fine to count all packets and remove + // this check. + if (packet.priority != Packet::Priority(0, 0)) { + continue; + } + size_t duration = packet.frame->Duration(); + if (duration > 0) { + last_duration = duration; // Save the most up-to-date (valid) duration. + } + } + num_samples += last_duration; + } + return num_samples; +} + +size_t PacketBuffer::GetSpanSamples(size_t last_decoded_length, + size_t sample_rate, + bool count_dtx_waiting_time) const { + if (buffer_.size() == 0) { + return 0; + } + + size_t span = buffer_.back().timestamp - buffer_.front().timestamp; + if (buffer_.back().frame && buffer_.back().frame->Duration() > 0) { + size_t duration = buffer_.back().frame->Duration(); + if (count_dtx_waiting_time && buffer_.back().frame->IsDtxPacket()) { + size_t waiting_time_samples = rtc::dchecked_cast( + buffer_.back().waiting_time->ElapsedMs() * (sample_rate / 1000)); + duration = std::max(duration, waiting_time_samples); + } + span += duration; + } else { + span += last_decoded_length; + } + return span; +} + +bool PacketBuffer::ContainsDtxOrCngPacket( + const DecoderDatabase* decoder_database) const { + RTC_DCHECK(decoder_database); + for (const Packet& packet : buffer_) { + if ((packet.frame && packet.frame->IsDtxPacket()) || + decoder_database->IsComfortNoise(packet.payload_type)) { + return true; + } + } + return false; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.h b/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.h new file mode 100644 index 0000000000..c6fb47ffbf --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.h @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_PACKET_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_PACKET_BUFFER_H_ + +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/packet.h" +#include "modules/include/module_common_types_public.h" // IsNewerTimestamp + +namespace webrtc { + +class DecoderDatabase; +class StatisticsCalculator; +class TickTimer; +struct SmartFlushingConfig { + // When calculating the flushing threshold, the maximum between the target + // level and this value is used. + int target_level_threshold_ms = 500; + // A smart flush is triggered when the packet buffer contains a multiple of + // the target level. + int target_level_multiplier = 3; +}; + +// This is the actual buffer holding the packets before decoding. +class PacketBuffer { + public: + enum BufferReturnCodes { + kOK = 0, + kFlushed, + kPartialFlush, + kNotFound, + kBufferEmpty, + kInvalidPacket, + kInvalidPointer + }; + + // Constructor creates a buffer which can hold a maximum of + // `max_number_of_packets` packets. + PacketBuffer(size_t max_number_of_packets, const TickTimer* tick_timer); + + // Deletes all packets in the buffer before destroying the buffer. + virtual ~PacketBuffer(); + + PacketBuffer(const PacketBuffer&) = delete; + PacketBuffer& operator=(const PacketBuffer&) = delete; + + // Flushes the buffer and deletes all packets in it. + virtual void Flush(StatisticsCalculator* stats); + + // Partial flush. Flush packets but leave some packets behind. + virtual void PartialFlush(int target_level_ms, + size_t sample_rate, + size_t last_decoded_length, + StatisticsCalculator* stats); + + // Returns true for an empty buffer. + virtual bool Empty() const; + + // Inserts `packet` into the buffer. The buffer will take over ownership of + // the packet object. + // Returns PacketBuffer::kOK on success, PacketBuffer::kFlushed if the buffer + // was flushed due to overfilling. + virtual int InsertPacket(Packet&& packet, + StatisticsCalculator* stats, + size_t last_decoded_length, + size_t sample_rate, + int target_level_ms, + const DecoderDatabase& decoder_database); + + // Inserts a list of packets into the buffer. The buffer will take over + // ownership of the packet objects. + // Returns PacketBuffer::kOK if all packets were inserted successfully. + // If the buffer was flushed due to overfilling, only a subset of the list is + // inserted, and PacketBuffer::kFlushed is returned. + // The last three parameters are included for legacy compatibility. + // TODO(hlundin): Redesign to not use current_*_payload_type and + // decoder_database. + virtual int InsertPacketList( + PacketList* packet_list, + const DecoderDatabase& decoder_database, + absl::optional* current_rtp_payload_type, + absl::optional* current_cng_rtp_payload_type, + StatisticsCalculator* stats, + size_t last_decoded_length, + size_t sample_rate, + int target_level_ms); + + // Gets the timestamp for the first packet in the buffer and writes it to the + // output variable `next_timestamp`. + // Returns PacketBuffer::kBufferEmpty if the buffer is empty, + // PacketBuffer::kOK otherwise. + virtual int NextTimestamp(uint32_t* next_timestamp) const; + + // Gets the timestamp for the first packet in the buffer with a timestamp no + // lower than the input limit `timestamp`. The result is written to the output + // variable `next_timestamp`. + // Returns PacketBuffer::kBufferEmpty if the buffer is empty, + // PacketBuffer::kOK otherwise. + virtual int NextHigherTimestamp(uint32_t timestamp, + uint32_t* next_timestamp) const; + + // Returns a (constant) pointer to the first packet in the buffer. Returns + // NULL if the buffer is empty. + virtual const Packet* PeekNextPacket() const; + + // Extracts the first packet in the buffer and returns it. + // Returns an empty optional if the buffer is empty. + virtual absl::optional GetNextPacket(); + + // Discards the first packet in the buffer. The packet is deleted. + // Returns PacketBuffer::kBufferEmpty if the buffer is empty, + // PacketBuffer::kOK otherwise. + virtual int DiscardNextPacket(StatisticsCalculator* stats); + + // Discards all packets that are (strictly) older than timestamp_limit, + // but newer than timestamp_limit - horizon_samples. Setting horizon_samples + // to zero implies that the horizon is set to half the timestamp range. That + // is, if a packet is more than 2^31 timestamps into the future compared with + // timestamp_limit (including wrap-around), it is considered old. + virtual void DiscardOldPackets(uint32_t timestamp_limit, + uint32_t horizon_samples, + StatisticsCalculator* stats); + + // Discards all packets that are (strictly) older than timestamp_limit. + virtual void DiscardAllOldPackets(uint32_t timestamp_limit, + StatisticsCalculator* stats); + + // Removes all packets with a specific payload type from the buffer. + virtual void DiscardPacketsWithPayloadType(uint8_t payload_type, + StatisticsCalculator* stats); + + // Returns the number of packets in the buffer, including duplicates and + // redundant packets. + virtual size_t NumPacketsInBuffer() const; + + // Returns the number of samples in the buffer, including samples carried in + // duplicate and redundant packets. + virtual size_t NumSamplesInBuffer(size_t last_decoded_length) const; + + // Returns the total duration in samples that the packets in the buffer spans + // across. + virtual size_t GetSpanSamples(size_t last_decoded_length, + size_t sample_rate, + bool count_dtx_waiting_time) const; + + // Returns true if the packet buffer contains any DTX or CNG packets. + virtual bool ContainsDtxOrCngPacket( + const DecoderDatabase* decoder_database) const; + + // Static method returning true if `timestamp` is older than `timestamp_limit` + // but less than `horizon_samples` behind `timestamp_limit`. For instance, + // with timestamp_limit = 100 and horizon_samples = 10, a timestamp in the + // range (90, 100) is considered obsolete, and will yield true. + // Setting `horizon_samples` to 0 is the same as setting it to 2^31, i.e., + // half the 32-bit timestamp range. + static bool IsObsoleteTimestamp(uint32_t timestamp, + uint32_t timestamp_limit, + uint32_t horizon_samples) { + return IsNewerTimestamp(timestamp_limit, timestamp) && + (horizon_samples == 0 || + IsNewerTimestamp(timestamp, timestamp_limit - horizon_samples)); + } + + private: + absl::optional smart_flushing_config_; + size_t max_number_of_packets_; + PacketList buffer_; + const TickTimer* tick_timer_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_PACKET_BUFFER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc new file mode 100644 index 0000000000..1a054daca3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc @@ -0,0 +1,989 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for PacketBuffer class. + +#include "modules/audio_coding/neteq/packet_buffer.h" + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/mock/mock_statistics_calculator.h" +#include "modules/audio_coding/neteq/packet.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::InSequence; +using ::testing::MockFunction; +using ::testing::Return; +using ::testing::StrictMock; + +namespace { +class MockEncodedAudioFrame : public webrtc::AudioDecoder::EncodedAudioFrame { + public: + MOCK_METHOD(size_t, Duration, (), (const, override)); + + MOCK_METHOD(bool, IsDtxPacket, (), (const, override)); + + MOCK_METHOD(absl::optional, + Decode, + (rtc::ArrayView decoded), + (const, override)); +}; + +// Helper class to generate packets. Packets must be deleted by the user. +class PacketGenerator { + public: + PacketGenerator(uint16_t seq_no, uint32_t ts, uint8_t pt, int frame_size); + virtual ~PacketGenerator() {} + void Reset(uint16_t seq_no, uint32_t ts, uint8_t pt, int frame_size); + webrtc::Packet NextPacket( + int payload_size_bytes, + std::unique_ptr audio_frame); + + uint16_t seq_no_; + uint32_t ts_; + uint8_t pt_; + int frame_size_; +}; + +PacketGenerator::PacketGenerator(uint16_t seq_no, + uint32_t ts, + uint8_t pt, + int frame_size) { + Reset(seq_no, ts, pt, frame_size); +} + +void PacketGenerator::Reset(uint16_t seq_no, + uint32_t ts, + uint8_t pt, + int frame_size) { + seq_no_ = seq_no; + ts_ = ts; + pt_ = pt; + frame_size_ = frame_size; +} + +webrtc::Packet PacketGenerator::NextPacket( + int payload_size_bytes, + std::unique_ptr audio_frame) { + webrtc::Packet packet; + packet.sequence_number = seq_no_; + packet.timestamp = ts_; + packet.payload_type = pt_; + packet.payload.SetSize(payload_size_bytes); + ++seq_no_; + ts_ += frame_size_; + packet.frame = std::move(audio_frame); + return packet; +} + +struct PacketsToInsert { + uint16_t sequence_number; + uint32_t timestamp; + uint8_t payload_type; + bool primary; + // Order of this packet to appear upon extraction, after inserting a series + // of packets. A negative number means that it should have been discarded + // before extraction. + int extract_order; +}; + +} // namespace + +namespace webrtc { + +// Start of test definitions. + +TEST(PacketBuffer, CreateAndDestroy) { + TickTimer tick_timer; + PacketBuffer* buffer = new PacketBuffer(10, &tick_timer); // 10 packets. + EXPECT_TRUE(buffer->Empty()); + delete buffer; +} + +TEST(PacketBuffer, InsertPacket) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(17u, 4711u, 0, 10); + StrictMock mock_stats; + MockDecoderDatabase decoder_database; + + const int payload_len = 100; + const Packet packet = gen.NextPacket(payload_len, nullptr); + EXPECT_EQ(0, buffer.InsertPacket(/*packet=*/packet.Clone(), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/10000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + uint32_t next_ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&next_ts)); + EXPECT_EQ(4711u, next_ts); + EXPECT_FALSE(buffer.Empty()); + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); + const Packet* next_packet = buffer.PeekNextPacket(); + EXPECT_EQ(packet, *next_packet); // Compare contents. + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. + + // Do not explicitly flush buffer or delete packet to test that it is deleted + // with the buffer. (Tested with Valgrind or similar tool.) +} + +// Test to flush buffer. +TEST(PacketBuffer, FlushBuffer) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + const int payload_len = 10; + StrictMock mock_stats; + MockDecoderDatabase decoder_database; + + // Insert 10 small packets; should be ok. + for (int i = 0; i < 10; ++i) { + EXPECT_EQ( + PacketBuffer::kOK, + buffer.InsertPacket(/*packet=*/gen.NextPacket(payload_len, nullptr), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + } + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + EXPECT_FALSE(buffer.Empty()); + + EXPECT_CALL(mock_stats, PacketsDiscarded(1)).Times(10); + buffer.Flush(&mock_stats); + // Buffer should delete the payloads itself. + EXPECT_EQ(0u, buffer.NumPacketsInBuffer()); + EXPECT_TRUE(buffer.Empty()); + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// Test to fill the buffer over the limits, and verify that it flushes. +TEST(PacketBuffer, OverfillBuffer) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + StrictMock mock_stats; + MockDecoderDatabase decoder_database; + + // Insert 10 small packets; should be ok. + const int payload_len = 10; + int i; + for (i = 0; i < 10; ++i) { + EXPECT_EQ( + PacketBuffer::kOK, + buffer.InsertPacket(/*packet=*/gen.NextPacket(payload_len, nullptr), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + } + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + uint32_t next_ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&next_ts)); + EXPECT_EQ(0u, next_ts); // Expect first inserted packet to be first in line. + + EXPECT_CALL(mock_stats, PacketsDiscarded(1)).Times(10); + const Packet packet = gen.NextPacket(payload_len, nullptr); + // Insert 11th packet; should flush the buffer and insert it after flushing. + EXPECT_EQ(PacketBuffer::kFlushed, + buffer.InsertPacket(/*packet=*/packet.Clone(), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&next_ts)); + // Expect last inserted packet to be first in line. + EXPECT_EQ(packet.timestamp, next_ts); + + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// Test a partial buffer flush. +TEST(PacketBuffer, PartialFlush) { + // Use a field trial to configure smart flushing. + test::ScopedFieldTrials field_trials( + "WebRTC-Audio-NetEqSmartFlushing/enabled:true," + "target_level_threshold_ms:0,target_level_multiplier:2/"); + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + const int payload_len = 10; + StrictMock mock_stats; + MockDecoderDatabase decoder_database; + + // Insert 10 small packets; should be ok. + for (int i = 0; i < 10; ++i) { + EXPECT_EQ( + PacketBuffer::kOK, + buffer.InsertPacket(/*packet=*/gen.NextPacket(payload_len, nullptr), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/100, + /*decoder_database=*/decoder_database)); + } + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + EXPECT_FALSE(buffer.Empty()); + + EXPECT_CALL(mock_stats, PacketsDiscarded(1)).Times(7); + buffer.PartialFlush(/*target_level_ms=*/30, + /*sample_rate=*/1000, + /*last_decoded_length=*/payload_len, + /*stats=*/&mock_stats); + // There should still be some packets left in the buffer. + EXPECT_EQ(3u, buffer.NumPacketsInBuffer()); + EXPECT_FALSE(buffer.Empty()); + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// Test to fill the buffer over the limits, and verify that the smart flush +// functionality works as expected. +TEST(PacketBuffer, SmartFlushOverfillBuffer) { + // Use a field trial to configure smart flushing. + test::ScopedFieldTrials field_trials( + "WebRTC-Audio-NetEqSmartFlushing/enabled:true," + "target_level_threshold_ms:0,target_level_multiplier:2/"); + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + StrictMock mock_stats; + MockDecoderDatabase decoder_database; + + // Insert 10 small packets; should be ok. + const int payload_len = 10; + int i; + for (i = 0; i < 10; ++i) { + EXPECT_EQ( + PacketBuffer::kOK, + buffer.InsertPacket(/*packet=*/gen.NextPacket(payload_len, nullptr), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/100, + /*decoder_database=*/decoder_database)); + } + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + uint32_t next_ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&next_ts)); + EXPECT_EQ(0u, next_ts); // Expect first inserted packet to be first in line. + + const Packet packet = gen.NextPacket(payload_len, nullptr); + EXPECT_CALL(mock_stats, PacketsDiscarded(1)).Times(6); + // Insert 11th packet; should cause a partial flush and insert the packet + // after flushing. + EXPECT_EQ(PacketBuffer::kPartialFlush, + buffer.InsertPacket(/*packet=*/packet.Clone(), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/40, + /*decoder_database=*/decoder_database)); + EXPECT_EQ(5u, buffer.NumPacketsInBuffer()); + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// Test inserting a list of packets. +TEST(PacketBuffer, InsertPacketList) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + PacketList list; + const int payload_len = 10; + + // Insert 10 small packets. + for (int i = 0; i < 10; ++i) { + list.push_back(gen.NextPacket(payload_len, nullptr)); + } + + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1), + absl::nullopt, factory.get()); + EXPECT_CALL(decoder_database, GetDecoderInfo(0)) + .WillRepeatedly(Return(&info)); + + StrictMock mock_stats; + + absl::optional current_pt; + absl::optional current_cng_pt; + EXPECT_EQ( + PacketBuffer::kOK, + buffer.InsertPacketList(/*packet_list=*/&list, + /*decoder_database=*/decoder_database, + /*current_rtp_payload_type=*/¤t_pt, + /*current_cng_rtp_payload_type=*/¤t_cng_pt, + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/30)); + EXPECT_TRUE(list.empty()); // The PacketBuffer should have depleted the list. + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + EXPECT_EQ(0, current_pt); // Current payload type changed to 0. + EXPECT_EQ(absl::nullopt, current_cng_pt); // CNG payload type not changed. + + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// Test inserting a list of packets. Last packet is of a different payload type. +// Expecting the buffer to flush. +// TODO(hlundin): Remove this test when legacy operation is no longer needed. +TEST(PacketBuffer, InsertPacketListChangePayloadType) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + PacketGenerator gen(0, 0, 0, 10); + PacketList list; + const int payload_len = 10; + + // Insert 10 small packets. + for (int i = 0; i < 10; ++i) { + list.push_back(gen.NextPacket(payload_len, nullptr)); + } + // Insert 11th packet of another payload type (not CNG). + { + Packet packet = gen.NextPacket(payload_len, nullptr); + packet.payload_type = 1; + list.push_back(std::move(packet)); + } + + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info0(SdpAudioFormat("pcmu", 8000, 1), + absl::nullopt, factory.get()); + EXPECT_CALL(decoder_database, GetDecoderInfo(0)) + .WillRepeatedly(Return(&info0)); + const DecoderDatabase::DecoderInfo info1(SdpAudioFormat("pcma", 8000, 1), + absl::nullopt, factory.get()); + EXPECT_CALL(decoder_database, GetDecoderInfo(1)) + .WillRepeatedly(Return(&info1)); + + StrictMock mock_stats; + + absl::optional current_pt; + absl::optional current_cng_pt; + EXPECT_CALL(mock_stats, PacketsDiscarded(1)).Times(10); + EXPECT_EQ( + PacketBuffer::kFlushed, + buffer.InsertPacketList(/*packet_list=*/&list, + /*decoder_database=*/decoder_database, + /*current_rtp_payload_type=*/¤t_pt, + /*current_cng_rtp_payload_type=*/¤t_cng_pt, + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/30)); + EXPECT_TRUE(list.empty()); // The PacketBuffer should have depleted the list. + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); // Only the last packet. + EXPECT_EQ(1, current_pt); // Current payload type changed to 1. + EXPECT_EQ(absl::nullopt, current_cng_pt); // CNG payload type not changed. + + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +TEST(PacketBuffer, ExtractOrderRedundancy) { + TickTimer tick_timer; + PacketBuffer buffer(100, &tick_timer); // 100 packets. + const int kPackets = 18; + const int kFrameSize = 10; + const int kPayloadLength = 10; + + PacketsToInsert packet_facts[kPackets] = { + {0xFFFD, 0xFFFFFFD7, 0, true, 0}, {0xFFFE, 0xFFFFFFE1, 0, true, 1}, + {0xFFFE, 0xFFFFFFD7, 1, false, -1}, {0xFFFF, 0xFFFFFFEB, 0, true, 2}, + {0xFFFF, 0xFFFFFFE1, 1, false, -1}, {0x0000, 0xFFFFFFF5, 0, true, 3}, + {0x0000, 0xFFFFFFEB, 1, false, -1}, {0x0001, 0xFFFFFFFF, 0, true, 4}, + {0x0001, 0xFFFFFFF5, 1, false, -1}, {0x0002, 0x0000000A, 0, true, 5}, + {0x0002, 0xFFFFFFFF, 1, false, -1}, {0x0003, 0x0000000A, 1, false, -1}, + {0x0004, 0x0000001E, 0, true, 7}, {0x0004, 0x00000014, 1, false, 6}, + {0x0005, 0x0000001E, 0, true, -1}, {0x0005, 0x00000014, 1, false, -1}, + {0x0006, 0x00000028, 0, true, 8}, {0x0006, 0x0000001E, 1, false, -1}, + }; + MockDecoderDatabase decoder_database; + + const size_t kExpectPacketsInBuffer = 9; + + std::vector expect_order(kExpectPacketsInBuffer); + + PacketGenerator gen(0, 0, 0, kFrameSize); + + StrictMock mock_stats; + + // Interleaving the EXPECT_CALL sequence with expectations on the MockFunction + // check ensures that exactly one call to PacketsDiscarded happens in each + // DiscardNextPacket call. + InSequence s; + MockFunction check; + for (int i = 0; i < kPackets; ++i) { + gen.Reset(packet_facts[i].sequence_number, packet_facts[i].timestamp, + packet_facts[i].payload_type, kFrameSize); + Packet packet = gen.NextPacket(kPayloadLength, nullptr); + packet.priority.codec_level = packet_facts[i].primary ? 0 : 1; + if (packet_facts[i].extract_order < 0) { + if (packet.priority.codec_level > 0) { + EXPECT_CALL(mock_stats, SecondaryPacketsDiscarded(1)); + } else { + EXPECT_CALL(mock_stats, PacketsDiscarded(1)); + } + } + EXPECT_CALL(check, Call(i)); + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacket(/*packet=*/packet.Clone(), + /*stats=*/&mock_stats, + /*last_decoded_length=*/kPayloadLength, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + if (packet_facts[i].extract_order >= 0) { + expect_order[packet_facts[i].extract_order] = std::move(packet); + } + check.Call(i); + } + + EXPECT_EQ(kExpectPacketsInBuffer, buffer.NumPacketsInBuffer()); + + for (size_t i = 0; i < kExpectPacketsInBuffer; ++i) { + const absl::optional packet = buffer.GetNextPacket(); + EXPECT_EQ(packet, expect_order[i]); // Compare contents. + } + EXPECT_TRUE(buffer.Empty()); + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +TEST(PacketBuffer, DiscardPackets) { + TickTimer tick_timer; + PacketBuffer buffer(100, &tick_timer); // 100 packets. + const uint16_t start_seq_no = 17; + const uint32_t start_ts = 4711; + const uint32_t ts_increment = 10; + PacketGenerator gen(start_seq_no, start_ts, 0, ts_increment); + PacketList list; + const int payload_len = 10; + StrictMock mock_stats; + MockDecoderDatabase decoder_database; + + constexpr int kTotalPackets = 10; + // Insert 10 small packets. + for (int i = 0; i < kTotalPackets; ++i) { + buffer.InsertPacket(/*packet=*/gen.NextPacket(payload_len, nullptr), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database); + } + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + + uint32_t current_ts = start_ts; + + // Discard them one by one and make sure that the right packets are at the + // front of the buffer. + constexpr int kDiscardPackets = 5; + + // Interleaving the EXPECT_CALL sequence with expectations on the MockFunction + // check ensures that exactly one call to PacketsDiscarded happens in each + // DiscardNextPacket call. + InSequence s; + MockFunction check; + for (int i = 0; i < kDiscardPackets; ++i) { + uint32_t ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&ts)); + EXPECT_EQ(current_ts, ts); + EXPECT_CALL(mock_stats, PacketsDiscarded(1)); + EXPECT_CALL(check, Call(i)); + EXPECT_EQ(PacketBuffer::kOK, buffer.DiscardNextPacket(&mock_stats)); + current_ts += ts_increment; + check.Call(i); + } + + constexpr int kRemainingPackets = kTotalPackets - kDiscardPackets; + // This will discard all remaining packets but one. The oldest packet is older + // than the indicated horizon_samples, and will thus be left in the buffer. + constexpr size_t kSkipPackets = 1; + EXPECT_CALL(mock_stats, PacketsDiscarded(1)) + .Times(kRemainingPackets - kSkipPackets); + EXPECT_CALL(check, Call(17)); // Arbitrary id number. + buffer.DiscardOldPackets(start_ts + kTotalPackets * ts_increment, + kRemainingPackets * ts_increment, &mock_stats); + check.Call(17); // Same arbitrary id number. + + EXPECT_EQ(kSkipPackets, buffer.NumPacketsInBuffer()); + uint32_t ts; + EXPECT_EQ(PacketBuffer::kOK, buffer.NextTimestamp(&ts)); + EXPECT_EQ(current_ts, ts); + + // Discard all remaining packets. + EXPECT_CALL(mock_stats, PacketsDiscarded(kSkipPackets)); + buffer.DiscardAllOldPackets(start_ts + kTotalPackets * ts_increment, + &mock_stats); + + EXPECT_TRUE(buffer.Empty()); + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +TEST(PacketBuffer, Reordering) { + TickTimer tick_timer; + PacketBuffer buffer(100, &tick_timer); // 100 packets. + const uint16_t start_seq_no = 17; + const uint32_t start_ts = 4711; + const uint32_t ts_increment = 10; + PacketGenerator gen(start_seq_no, start_ts, 0, ts_increment); + const int payload_len = 10; + + // Generate 10 small packets and insert them into a PacketList. Insert every + // odd packet to the front, and every even packet to the back, thus creating + // a (rather strange) reordering. + PacketList list; + for (int i = 0; i < 10; ++i) { + Packet packet = gen.NextPacket(payload_len, nullptr); + if (i % 2) { + list.push_front(std::move(packet)); + } else { + list.push_back(std::move(packet)); + } + } + + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1), + absl::nullopt, factory.get()); + EXPECT_CALL(decoder_database, GetDecoderInfo(0)) + .WillRepeatedly(Return(&info)); + absl::optional current_pt; + absl::optional current_cng_pt; + + StrictMock mock_stats; + + EXPECT_EQ( + PacketBuffer::kOK, + buffer.InsertPacketList(/*packet_list=*/&list, + /*decoder_database=*/decoder_database, + /*current_rtp_payload_type=*/¤t_pt, + /*current_cng_rtp_payload_type=*/¤t_cng_pt, + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/30)); + EXPECT_EQ(10u, buffer.NumPacketsInBuffer()); + + // Extract them and make sure that come out in the right order. + uint32_t current_ts = start_ts; + for (int i = 0; i < 10; ++i) { + const absl::optional packet = buffer.GetNextPacket(); + ASSERT_TRUE(packet); + EXPECT_EQ(current_ts, packet->timestamp); + current_ts += ts_increment; + } + EXPECT_TRUE(buffer.Empty()); + + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// The test first inserts a packet with narrow-band CNG, then a packet with +// wide-band speech. The expected behavior of the packet buffer is to detect a +// change in sample rate, even though no speech packet has been inserted before, +// and flush out the CNG packet. +TEST(PacketBuffer, CngFirstThenSpeechWithNewSampleRate) { + TickTimer tick_timer; + PacketBuffer buffer(10, &tick_timer); // 10 packets. + const uint8_t kCngPt = 13; + const int kPayloadLen = 10; + const uint8_t kSpeechPt = 100; + + MockDecoderDatabase decoder_database; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info_cng(SdpAudioFormat("cn", 8000, 1), + absl::nullopt, factory.get()); + EXPECT_CALL(decoder_database, GetDecoderInfo(kCngPt)) + .WillRepeatedly(Return(&info_cng)); + const DecoderDatabase::DecoderInfo info_speech( + SdpAudioFormat("l16", 16000, 1), absl::nullopt, factory.get()); + EXPECT_CALL(decoder_database, GetDecoderInfo(kSpeechPt)) + .WillRepeatedly(Return(&info_speech)); + + // Insert first packet, which is narrow-band CNG. + PacketGenerator gen(0, 0, kCngPt, 10); + PacketList list; + list.push_back(gen.NextPacket(kPayloadLen, nullptr)); + absl::optional current_pt; + absl::optional current_cng_pt; + + StrictMock mock_stats; + + EXPECT_EQ( + PacketBuffer::kOK, + buffer.InsertPacketList(/*packet_list=*/&list, + /*decoder_database=*/decoder_database, + /*current_rtp_payload_type=*/¤t_pt, + /*current_cng_rtp_payload_type=*/¤t_cng_pt, + /*stats=*/&mock_stats, + /*last_decoded_length=*/kPayloadLen, + /*sample_rate=*/1000, + /*target_level_ms=*/30)); + EXPECT_TRUE(list.empty()); + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); + ASSERT_TRUE(buffer.PeekNextPacket()); + EXPECT_EQ(kCngPt, buffer.PeekNextPacket()->payload_type); + EXPECT_EQ(current_pt, absl::nullopt); // Current payload type not set. + EXPECT_EQ(kCngPt, current_cng_pt); // CNG payload type set. + + // Insert second packet, which is wide-band speech. + { + Packet packet = gen.NextPacket(kPayloadLen, nullptr); + packet.payload_type = kSpeechPt; + list.push_back(std::move(packet)); + } + // Expect the buffer to flush out the CNG packet, since it does not match the + // new speech sample rate. + EXPECT_CALL(mock_stats, PacketsDiscarded(1)); + EXPECT_EQ( + PacketBuffer::kFlushed, + buffer.InsertPacketList(/*packet_list=*/&list, + /*decoder_database=*/decoder_database, + /*current_rtp_payload_type=*/¤t_pt, + /*current_cng_rtp_payload_type=*/¤t_cng_pt, + /*stats=*/&mock_stats, + /*last_decoded_length=*/kPayloadLen, + /*sample_rate=*/1000, + /*target_level_ms=*/30)); + EXPECT_TRUE(list.empty()); + EXPECT_EQ(1u, buffer.NumPacketsInBuffer()); + ASSERT_TRUE(buffer.PeekNextPacket()); + EXPECT_EQ(kSpeechPt, buffer.PeekNextPacket()->payload_type); + + EXPECT_EQ(kSpeechPt, current_pt); // Current payload type set. + EXPECT_EQ(absl::nullopt, current_cng_pt); // CNG payload type reset. + + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +TEST(PacketBuffer, Failures) { + const uint16_t start_seq_no = 17; + const uint32_t start_ts = 4711; + const uint32_t ts_increment = 10; + int payload_len = 100; + PacketGenerator gen(start_seq_no, start_ts, 0, ts_increment); + TickTimer tick_timer; + StrictMock mock_stats; + MockDecoderDatabase decoder_database; + + PacketBuffer* buffer = new PacketBuffer(100, &tick_timer); // 100 packets. + { + Packet packet = gen.NextPacket(payload_len, nullptr); + packet.payload.Clear(); + EXPECT_EQ(PacketBuffer::kInvalidPacket, + buffer->InsertPacket(/*packet=*/std::move(packet), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + } + // Buffer should still be empty. Test all empty-checks. + uint32_t temp_ts; + EXPECT_EQ(PacketBuffer::kBufferEmpty, buffer->NextTimestamp(&temp_ts)); + EXPECT_EQ(PacketBuffer::kBufferEmpty, + buffer->NextHigherTimestamp(0, &temp_ts)); + EXPECT_EQ(NULL, buffer->PeekNextPacket()); + EXPECT_FALSE(buffer->GetNextPacket()); + + // Discarding packets will not invoke mock_stats.PacketDiscarded() because the + // packet buffer is empty. + EXPECT_EQ(PacketBuffer::kBufferEmpty, buffer->DiscardNextPacket(&mock_stats)); + buffer->DiscardAllOldPackets(0, &mock_stats); + + // Insert one packet to make the buffer non-empty. + EXPECT_EQ( + PacketBuffer::kOK, + buffer->InsertPacket(/*packet=*/gen.NextPacket(payload_len, nullptr), + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + EXPECT_EQ(PacketBuffer::kInvalidPointer, buffer->NextTimestamp(NULL)); + EXPECT_EQ(PacketBuffer::kInvalidPointer, + buffer->NextHigherTimestamp(0, NULL)); + delete buffer; + + // Insert packet list of three packets, where the second packet has an invalid + // payload. Expect first packet to be inserted, and the remaining two to be + // discarded. + buffer = new PacketBuffer(100, &tick_timer); // 100 packets. + PacketList list; + list.push_back(gen.NextPacket(payload_len, nullptr)); // Valid packet. + { + Packet packet = gen.NextPacket(payload_len, nullptr); + packet.payload.Clear(); // Invalid. + list.push_back(std::move(packet)); + } + list.push_back(gen.NextPacket(payload_len, nullptr)); // Valid packet. + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1), + absl::nullopt, factory.get()); + EXPECT_CALL(decoder_database, GetDecoderInfo(0)) + .WillRepeatedly(Return(&info)); + absl::optional current_pt; + absl::optional current_cng_pt; + EXPECT_EQ( + PacketBuffer::kInvalidPacket, + buffer->InsertPacketList(/*packet_list=*/&list, + /*decoder_database=*/decoder_database, + /*current_rtp_payload_type=*/¤t_pt, + /*current_cng_rtp_payload_type=*/¤t_cng_pt, + /*stats=*/&mock_stats, + /*last_decoded_length=*/payload_len, + /*sample_rate=*/1000, + /*target_level_ms=*/30)); + EXPECT_TRUE(list.empty()); // The PacketBuffer should have depleted the list. + EXPECT_EQ(1u, buffer->NumPacketsInBuffer()); + delete buffer; + EXPECT_CALL(decoder_database, Die()); // Called when object is deleted. +} + +// Test packet comparison function. +// The function should return true if the first packet "goes before" the second. +TEST(PacketBuffer, ComparePackets) { + PacketGenerator gen(0, 0, 0, 10); + Packet a(gen.NextPacket(10, nullptr)); // SN = 0, TS = 0. + Packet b(gen.NextPacket(10, nullptr)); // SN = 1, TS = 10. + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_TRUE(a < b); + EXPECT_FALSE(a > b); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(a >= b); + + // Testing wrap-around case; 'a' is earlier but has a larger timestamp value. + a.timestamp = 0xFFFFFFFF - 10; + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_TRUE(a < b); + EXPECT_FALSE(a > b); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(a >= b); + + // Test equal packets. + EXPECT_TRUE(a == a); + EXPECT_FALSE(a != a); + EXPECT_FALSE(a < a); + EXPECT_FALSE(a > a); + EXPECT_TRUE(a <= a); + EXPECT_TRUE(a >= a); + + // Test equal timestamps but different sequence numbers (0 and 1). + a.timestamp = b.timestamp; + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_TRUE(a < b); + EXPECT_FALSE(a > b); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(a >= b); + + // Test equal timestamps but different sequence numbers (32767 and 1). + a.sequence_number = 0xFFFF; + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_TRUE(a < b); + EXPECT_FALSE(a > b); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(a >= b); + + // Test equal timestamps and sequence numbers, but differing priorities. + a.sequence_number = b.sequence_number; + a.priority = {1, 0}; + b.priority = {0, 0}; + // a after b + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + EXPECT_FALSE(a < b); + EXPECT_TRUE(a > b); + EXPECT_FALSE(a <= b); + EXPECT_TRUE(a >= b); + + Packet c(gen.NextPacket(0, nullptr)); // SN = 2, TS = 20. + Packet d(gen.NextPacket(0, nullptr)); // SN = 3, TS = 20. + c.timestamp = b.timestamp; + d.timestamp = b.timestamp; + c.sequence_number = b.sequence_number; + d.sequence_number = b.sequence_number; + c.priority = {1, 1}; + d.priority = {0, 1}; + // c after d + EXPECT_FALSE(c == d); + EXPECT_TRUE(c != d); + EXPECT_FALSE(c < d); + EXPECT_TRUE(c > d); + EXPECT_FALSE(c <= d); + EXPECT_TRUE(c >= d); + + // c after a + EXPECT_FALSE(c == a); + EXPECT_TRUE(c != a); + EXPECT_FALSE(c < a); + EXPECT_TRUE(c > a); + EXPECT_FALSE(c <= a); + EXPECT_TRUE(c >= a); + + // c after b + EXPECT_FALSE(c == b); + EXPECT_TRUE(c != b); + EXPECT_FALSE(c < b); + EXPECT_TRUE(c > b); + EXPECT_FALSE(c <= b); + EXPECT_TRUE(c >= b); + + // a after d + EXPECT_FALSE(a == d); + EXPECT_TRUE(a != d); + EXPECT_FALSE(a < d); + EXPECT_TRUE(a > d); + EXPECT_FALSE(a <= d); + EXPECT_TRUE(a >= d); + + // d after b + EXPECT_FALSE(d == b); + EXPECT_TRUE(d != b); + EXPECT_FALSE(d < b); + EXPECT_TRUE(d > b); + EXPECT_FALSE(d <= b); + EXPECT_TRUE(d >= b); +} + +TEST(PacketBuffer, GetSpanSamples) { + constexpr size_t kFrameSizeSamples = 10; + constexpr int kPayloadSizeBytes = 1; // Does not matter to this test; + constexpr uint32_t kStartTimeStamp = 0xFFFFFFFE; // Close to wrap around. + constexpr int kSampleRateHz = 48000; + constexpr bool KCountDtxWaitingTime = false; + TickTimer tick_timer; + PacketBuffer buffer(3, &tick_timer); + PacketGenerator gen(0, kStartTimeStamp, 0, kFrameSizeSamples); + StrictMock mock_stats; + MockDecoderDatabase decoder_database; + + Packet packet_1 = gen.NextPacket(kPayloadSizeBytes, nullptr); + + std::unique_ptr mock_audio_frame = + std::make_unique(); + EXPECT_CALL(*mock_audio_frame, Duration()) + .WillRepeatedly(Return(kFrameSizeSamples)); + Packet packet_2 = + gen.NextPacket(kPayloadSizeBytes, std::move(mock_audio_frame)); + + RTC_DCHECK_GT(packet_1.timestamp, + packet_2.timestamp); // Tmestamp wrapped around. + + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacket(/*packet=*/std::move(packet_1), + /*stats=*/&mock_stats, + /*last_decoded_length=*/kFrameSizeSamples, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + + constexpr size_t kLastDecodedSizeSamples = 2; + // packet_1 has no access to duration, and relies last decoded duration as + // input. + EXPECT_EQ(kLastDecodedSizeSamples, + buffer.GetSpanSamples(kLastDecodedSizeSamples, kSampleRateHz, + KCountDtxWaitingTime)); + + EXPECT_EQ(PacketBuffer::kOK, + buffer.InsertPacket(/*packet=*/std::move(packet_2), + /*stats=*/&mock_stats, + /*last_decoded_length=*/kFrameSizeSamples, + /*sample_rate=*/1000, + /*target_level_ms=*/60, + /*decoder_database=*/decoder_database)); + + EXPECT_EQ(kFrameSizeSamples * 2, + buffer.GetSpanSamples(0, kSampleRateHz, KCountDtxWaitingTime)); + + // packet_2 has access to duration, and ignores last decoded duration as + // input. + EXPECT_EQ(kFrameSizeSamples * 2, + buffer.GetSpanSamples(kLastDecodedSizeSamples, kSampleRateHz, + KCountDtxWaitingTime)); +} + +namespace { +void TestIsObsoleteTimestamp(uint32_t limit_timestamp) { + // Check with zero horizon, which implies that the horizon is at 2^31, i.e., + // half the timestamp range. + static const uint32_t kZeroHorizon = 0; + static const uint32_t k2Pow31Minus1 = 0x7FFFFFFF; + // Timestamp on the limit is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp, limit_timestamp, kZeroHorizon)); + // 1 sample behind is old. + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp(limit_timestamp - 1, + limit_timestamp, kZeroHorizon)); + // 2^31 - 1 samples behind is old. + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp(limit_timestamp - k2Pow31Minus1, + limit_timestamp, kZeroHorizon)); + // 1 sample ahead is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp + 1, limit_timestamp, kZeroHorizon)); + // If |t1-t2|=2^31 and t1>t2, t2 is older than t1 but not the opposite. + uint32_t other_timestamp = limit_timestamp + (1 << 31); + uint32_t lowest_timestamp = std::min(limit_timestamp, other_timestamp); + uint32_t highest_timestamp = std::max(limit_timestamp, other_timestamp); + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp( + lowest_timestamp, highest_timestamp, kZeroHorizon)); + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + highest_timestamp, lowest_timestamp, kZeroHorizon)); + + // Fixed horizon at 10 samples. + static const uint32_t kHorizon = 10; + // Timestamp on the limit is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp(limit_timestamp, + limit_timestamp, kHorizon)); + // 1 sample behind is old. + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp(limit_timestamp - 1, + limit_timestamp, kHorizon)); + // 9 samples behind is old. + EXPECT_TRUE(PacketBuffer::IsObsoleteTimestamp(limit_timestamp - 9, + limit_timestamp, kHorizon)); + // 10 samples behind is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp(limit_timestamp - 10, + limit_timestamp, kHorizon)); + // 2^31 - 1 samples behind is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp( + limit_timestamp - k2Pow31Minus1, limit_timestamp, kHorizon)); + // 1 sample ahead is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp(limit_timestamp + 1, + limit_timestamp, kHorizon)); + // 2^31 samples ahead is not old. + EXPECT_FALSE(PacketBuffer::IsObsoleteTimestamp(limit_timestamp + (1 << 31), + limit_timestamp, kHorizon)); +} +} // namespace + +// Test the IsObsoleteTimestamp method with different limit timestamps. +TEST(PacketBuffer, IsObsoleteTimestamp) { + TestIsObsoleteTimestamp(0); + TestIsObsoleteTimestamp(1); + TestIsObsoleteTimestamp(0xFFFFFFFF); // -1 in uint32_t. + TestIsObsoleteTimestamp(0x80000000); // 2^31. + TestIsObsoleteTimestamp(0x80000001); // 2^31 + 1. + TestIsObsoleteTimestamp(0x7FFFFFFF); // 2^31 - 1. +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.cc b/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.cc new file mode 100644 index 0000000000..9999d6764b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/post_decode_vad.h" + +namespace webrtc { + +PostDecodeVad::~PostDecodeVad() { + if (vad_instance_) + WebRtcVad_Free(vad_instance_); +} + +void PostDecodeVad::Enable() { + if (!vad_instance_) { + // Create the instance. + vad_instance_ = WebRtcVad_Create(); + if (vad_instance_ == nullptr) { + // Failed to create instance. + Disable(); + return; + } + } + Init(); + enabled_ = true; +} + +void PostDecodeVad::Disable() { + enabled_ = false; + running_ = false; +} + +void PostDecodeVad::Init() { + running_ = false; + if (vad_instance_) { + WebRtcVad_Init(vad_instance_); + WebRtcVad_set_mode(vad_instance_, kVadMode); + running_ = true; + } +} + +void PostDecodeVad::Update(int16_t* signal, + size_t length, + AudioDecoder::SpeechType speech_type, + bool sid_frame, + int fs_hz) { + if (!vad_instance_ || !enabled_) { + return; + } + + if (speech_type == AudioDecoder::kComfortNoise || sid_frame || + fs_hz > 16000) { + // TODO(hlundin): Remove restriction on fs_hz. + running_ = false; + active_speech_ = true; + sid_interval_counter_ = 0; + } else if (!running_) { + ++sid_interval_counter_; + } + + if (sid_interval_counter_ >= kVadAutoEnable) { + Init(); + } + + if (length > 0 && running_) { + size_t vad_sample_index = 0; + active_speech_ = false; + // Loop through frame sizes 30, 20, and 10 ms. + for (int vad_frame_size_ms = 30; vad_frame_size_ms >= 10; + vad_frame_size_ms -= 10) { + size_t vad_frame_size_samples = + static_cast(vad_frame_size_ms * fs_hz / 1000); + while (length - vad_sample_index >= vad_frame_size_samples) { + int vad_return = + WebRtcVad_Process(vad_instance_, fs_hz, &signal[vad_sample_index], + vad_frame_size_samples); + active_speech_ |= (vad_return == 1); + vad_sample_index += vad_frame_size_samples; + } + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.h b/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.h new file mode 100644 index 0000000000..3bd91b9edb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_POST_DECODE_VAD_H_ +#define MODULES_AUDIO_CODING_NETEQ_POST_DECODE_VAD_H_ + +#include +#include + +#include "api/audio_codecs/audio_decoder.h" +#include "common_audio/vad/include/webrtc_vad.h" + +namespace webrtc { + +class PostDecodeVad { + public: + PostDecodeVad() + : enabled_(false), + running_(false), + active_speech_(true), + sid_interval_counter_(0), + vad_instance_(NULL) {} + + virtual ~PostDecodeVad(); + + PostDecodeVad(const PostDecodeVad&) = delete; + PostDecodeVad& operator=(const PostDecodeVad&) = delete; + + // Enables post-decode VAD. + void Enable(); + + // Disables post-decode VAD. + void Disable(); + + // Initializes post-decode VAD. + void Init(); + + // Updates post-decode VAD with the audio data in `signal` having `length` + // samples. The data is of type `speech_type`, at the sample rate `fs_hz`. + void Update(int16_t* signal, + size_t length, + AudioDecoder::SpeechType speech_type, + bool sid_frame, + int fs_hz); + + // Accessors. + bool enabled() const { return enabled_; } + bool running() const { return running_; } + bool active_speech() const { return active_speech_; } + + private: + static const int kVadMode = 0; // Sets aggressiveness to "Normal". + // Number of Update() calls without CNG/SID before re-enabling VAD. + static const int kVadAutoEnable = 3000; + + bool enabled_; + bool running_; + bool active_speech_; + int sid_interval_counter_; + ::VadInst* vad_instance_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_POST_DECODE_VAD_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad_unittest.cc new file mode 100644 index 0000000000..da3e4e864e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad_unittest.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for PostDecodeVad class. + +#include "modules/audio_coding/neteq/post_decode_vad.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(PostDecodeVad, CreateAndDestroy) { + PostDecodeVad vad; +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.cc b/third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.cc new file mode 100644 index 0000000000..232170b177 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.cc @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/preemptive_expand.h" + +#include + +#include "api/array_view.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/time_stretch.h" + +namespace webrtc { + +PreemptiveExpand::ReturnCodes PreemptiveExpand::Process( + const int16_t* input, + size_t input_length, + size_t old_data_length, + AudioMultiVector* output, + size_t* length_change_samples) { + old_data_length_per_channel_ = old_data_length; + // Input length must be (almost) 30 ms. + // Also, the new part must be at least `overlap_samples_` elements. + static const size_t k15ms = 120; // 15 ms = 120 samples at 8 kHz sample rate. + if (num_channels_ == 0 || + input_length / num_channels_ < (2 * k15ms - 1) * fs_mult_ || + old_data_length >= input_length / num_channels_ - overlap_samples_) { + // Length of input data too short to do preemptive expand. Simply move all + // data from input to output. + output->PushBackInterleaved( + rtc::ArrayView(input, input_length)); + return kError; + } + const bool kFastMode = false; // Fast mode is not available for PE Expand. + return TimeStretch::Process(input, input_length, kFastMode, output, + length_change_samples); +} + +void PreemptiveExpand::SetParametersForPassiveSpeech(size_t len, + int16_t* best_correlation, + size_t* peak_index) const { + // When the signal does not contain any active speech, the correlation does + // not matter. Simply set it to zero. + *best_correlation = 0; + + // For low energy expansion, the new data can be less than 15 ms, + // but we must ensure that best_correlation is not larger than the length of + // the new data. + // but we must ensure that best_correlation is not larger than the new data. + *peak_index = std::min(*peak_index, len - old_data_length_per_channel_); +} + +PreemptiveExpand::ReturnCodes PreemptiveExpand::CheckCriteriaAndStretch( + const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool /*fast_mode*/, + AudioMultiVector* output) const { + // Pre-calculate common multiplication with `fs_mult_`. + // 120 corresponds to 15 ms. + size_t fs_mult_120 = static_cast(fs_mult_ * 120); + // Check for strong correlation (>0.9 in Q14) and at least 15 ms new data, + // or passive speech. + if (((best_correlation > kCorrelationThreshold) && + (old_data_length_per_channel_ <= fs_mult_120)) || + !active_speech) { + // Do accelerate operation by overlap add. + + // Set length of the first part, not to be modified. + size_t unmodified_length = + std::max(old_data_length_per_channel_, fs_mult_120); + // Copy first part, including cross-fade region. + output->PushBackInterleaved(rtc::ArrayView( + input, (unmodified_length + peak_index) * num_channels_)); + // Copy the last `peak_index` samples up to 15 ms to `temp_vector`. + AudioMultiVector temp_vector(num_channels_); + temp_vector.PushBackInterleaved(rtc::ArrayView( + &input[(unmodified_length - peak_index) * num_channels_], + peak_index * num_channels_)); + // Cross-fade `temp_vector` onto the end of `output`. + output->CrossFade(temp_vector, peak_index); + // Copy the last unmodified part, 15 ms + pitch period until the end. + output->PushBackInterleaved(rtc::ArrayView( + &input[unmodified_length * num_channels_], + input_length - unmodified_length * num_channels_)); + + if (active_speech) { + return kSuccess; + } else { + return kSuccessLowEnergy; + } + } else { + // Accelerate not allowed. Simply move all data from decoded to outData. + output->PushBackInterleaved( + rtc::ArrayView(input, input_length)); + return kNoStretch; + } +} + +PreemptiveExpand* PreemptiveExpandFactory::Create( + int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise, + size_t overlap_samples) const { + return new PreemptiveExpand(sample_rate_hz, num_channels, background_noise, + overlap_samples); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.h b/third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.h new file mode 100644 index 0000000000..6338b993fd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_PREEMPTIVE_EXPAND_H_ +#define MODULES_AUDIO_CODING_NETEQ_PREEMPTIVE_EXPAND_H_ + +#include +#include + +#include "modules/audio_coding/neteq/time_stretch.h" + +namespace webrtc { + +class AudioMultiVector; +class BackgroundNoise; + +// This class implements the PreemptiveExpand operation. Most of the work is +// done in the base class TimeStretch, which is shared with the Accelerate +// operation. In the PreemptiveExpand class, the operations that are specific to +// PreemptiveExpand are implemented. +class PreemptiveExpand : public TimeStretch { + public: + PreemptiveExpand(int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise, + size_t overlap_samples) + : TimeStretch(sample_rate_hz, num_channels, background_noise), + old_data_length_per_channel_(0), + overlap_samples_(overlap_samples) {} + + PreemptiveExpand(const PreemptiveExpand&) = delete; + PreemptiveExpand& operator=(const PreemptiveExpand&) = delete; + + // This method performs the actual PreemptiveExpand operation. The samples are + // read from `input`, of length `input_length` elements, and are written to + // `output`. The number of samples added through time-stretching is + // is provided in the output `length_change_samples`. The method returns + // the outcome of the operation as an enumerator value. + ReturnCodes Process(const int16_t* pw16_decoded, + size_t len, + size_t old_data_len, + AudioMultiVector* output, + size_t* length_change_samples); + + protected: + // Sets the parameters `best_correlation` and `peak_index` to suitable + // values when the signal contains no active speech. + void SetParametersForPassiveSpeech(size_t input_length, + int16_t* best_correlation, + size_t* peak_index) const override; + + // Checks the criteria for performing the time-stretching operation and, + // if possible, performs the time-stretching. + ReturnCodes CheckCriteriaAndStretch(const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool /*fast_mode*/, + AudioMultiVector* output) const override; + + private: + size_t old_data_length_per_channel_; + size_t overlap_samples_; +}; + +struct PreemptiveExpandFactory { + PreemptiveExpandFactory() {} + virtual ~PreemptiveExpandFactory() {} + + virtual PreemptiveExpand* Create(int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise, + size_t overlap_samples) const; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_PREEMPTIVE_EXPAND_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/random_vector.cc b/third_party/libwebrtc/modules/audio_coding/neteq/random_vector.cc new file mode 100644 index 0000000000..ada175831c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/random_vector.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/random_vector.h" + +namespace webrtc { + +const int16_t RandomVector::kRandomTable[RandomVector::kRandomTableSize] = { + 2680, 5532, 441, 5520, 16170, -5146, -1024, -8733, 3115, + 9598, -10380, -4959, -1280, -21716, 7133, -1522, 13458, -3902, + 2789, -675, 3441, 5016, -13599, -4003, -2739, 3922, -7209, + 13352, -11617, -7241, 12905, -2314, 5426, 10121, -9702, 11207, + -13542, 1373, 816, -5934, -12504, 4798, 1811, 4112, -613, + 201, -10367, -2960, -2419, 3442, 4299, -6116, -6092, 1552, + -1650, -480, -1237, 18720, -11858, -8303, -8212, 865, -2890, + -16968, 12052, -5845, -5912, 9777, -5665, -6294, 5426, -4737, + -6335, 1652, 761, 3832, 641, -8552, -9084, -5753, 8146, + 12156, -4915, 15086, -1231, -1869, 11749, -9319, -6403, 11407, + 6232, -1683, 24340, -11166, 4017, -10448, 3153, -2936, 6212, + 2891, -866, -404, -4807, -2324, -1917, -2388, -6470, -3895, + -10300, 5323, -5403, 2205, 4640, 7022, -21186, -6244, -882, + -10031, -3395, -12885, 7155, -5339, 5079, -2645, -9515, 6622, + 14651, 15852, 359, 122, 8246, -3502, -6696, -3679, -13535, + -1409, -704, -7403, -4007, 1798, 279, -420, -12796, -14219, + 1141, 3359, 11434, 7049, -6684, -7473, 14283, -4115, -9123, + -8969, 4152, 4117, 13792, 5742, 16168, 8661, -1609, -6095, + 1881, 14380, -5588, 6758, -6425, -22969, -7269, 7031, 1119, + -1611, -5850, -11281, 3559, -8952, -10146, -4667, -16251, -1538, + 2062, -1012, -13073, 227, -3142, -5265, 20, 5770, -7559, + 4740, -4819, 992, -8208, -7130, -4652, 6725, 7369, -1036, + 13144, -1588, -5304, -2344, -449, -5705, -8894, 5205, -17904, + -11188, -1022, 4852, 10101, -5255, -4200, -752, 7941, -1543, + 5959, 14719, 13346, 17045, -15605, -1678, -1600, -9230, 68, + 23348, 1172, 7750, 11212, -18227, 9956, 4161, 883, 3947, + 4341, 1014, -4889, -2603, 1246, -5630, -3596, -870, -1298, + 2784, -3317, -6612, -20541, 4166, 4181, -8625, 3562, 12890, + 4761, 3205, -12259, -8579}; + +void RandomVector::Reset() { + seed_ = 777; + seed_increment_ = 1; +} + +void RandomVector::Generate(size_t length, int16_t* output) { + for (size_t i = 0; i < length; i++) { + seed_ += seed_increment_; + size_t position = seed_ & (kRandomTableSize - 1); + output[i] = kRandomTable[position]; + } +} + +void RandomVector::IncreaseSeedIncrement(int16_t increase_by) { + seed_increment_ += increase_by; + seed_increment_ &= kRandomTableSize - 1; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/random_vector.h b/third_party/libwebrtc/modules/audio_coding/neteq/random_vector.h new file mode 100644 index 0000000000..4a782f1116 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/random_vector.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_RANDOM_VECTOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_RANDOM_VECTOR_H_ + +#include +#include + +namespace webrtc { + +// This class generates pseudo-random samples. +class RandomVector { + public: + static const size_t kRandomTableSize = 256; + static const int16_t kRandomTable[kRandomTableSize]; + + RandomVector() : seed_(777), seed_increment_(1) {} + + RandomVector(const RandomVector&) = delete; + RandomVector& operator=(const RandomVector&) = delete; + + void Reset(); + + void Generate(size_t length, int16_t* output); + + void IncreaseSeedIncrement(int16_t increase_by); + + // Accessors and mutators. + int16_t seed_increment() { return seed_increment_; } + void set_seed_increment(int16_t value) { seed_increment_ = value; } + + private: + uint32_t seed_; + int16_t seed_increment_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_RANDOM_VECTOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/random_vector_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/random_vector_unittest.cc new file mode 100644 index 0000000000..44479a6dd6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/random_vector_unittest.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for RandomVector class. + +#include "modules/audio_coding/neteq/random_vector.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(RandomVector, CreateAndDestroy) { + RandomVector random_vector; +} + +// TODO(hlundin): Write more tests. + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.cc b/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.cc new file mode 100644 index 0000000000..cec9f2f8a0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.cc @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/red_payload_splitter.h" + +#include + +#include +#include +#include +#include + +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/packet.h" +#include "rtc_base/buffer.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +// The method loops through a list of packets {A, B, C, ...}. Each packet is +// split into its corresponding RED payloads, {A1, A2, ...}, which is +// temporarily held in the list `new_packets`. +// When the first packet in `packet_list` has been processed, the original +// packet is replaced by the new ones in `new_packets`, so that `packet_list` +// becomes: {A1, A2, ..., B, C, ...}. The method then continues with B, and C, +// until all the original packets have been replaced by their split payloads. +bool RedPayloadSplitter::SplitRed(PacketList* packet_list) { + // Too many RED blocks indicates that something is wrong. Clamp it at some + // reasonable value. + const size_t kMaxRedBlocks = 32; + bool ret = true; + PacketList::iterator it = packet_list->begin(); + while (it != packet_list->end()) { + const Packet& red_packet = *it; + RTC_DCHECK(!red_packet.payload.empty()); + const uint8_t* payload_ptr = red_packet.payload.data(); + size_t payload_length = red_packet.payload.size(); + + // Read RED headers (according to RFC 2198): + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |F| block PT | timestamp offset | block length | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Last RED header: + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // |0| Block PT | + // +-+-+-+-+-+-+-+-+ + + struct RedHeader { + uint8_t payload_type; + uint32_t timestamp; + size_t payload_length; + }; + + std::vector new_headers; + bool last_block = false; + size_t sum_length = 0; + while (!last_block) { + if (payload_length == 0) { + RTC_LOG(LS_WARNING) << "SplitRed header too short"; + return false; + } + RedHeader new_header; + // Check the F bit. If F == 0, this was the last block. + last_block = ((*payload_ptr & 0x80) == 0); + // Bits 1 through 7 are payload type. + new_header.payload_type = payload_ptr[0] & 0x7F; + if (last_block) { + // No more header data to read. + sum_length += kRedLastHeaderLength; // Account for RED header size. + new_header.timestamp = red_packet.timestamp; + new_header.payload_length = red_packet.payload.size() - sum_length; + payload_ptr += kRedLastHeaderLength; // Advance to first payload byte. + payload_length -= kRedLastHeaderLength; + } else { + if (payload_length < kRedHeaderLength) { + RTC_LOG(LS_WARNING) << "SplitRed header too short"; + return false; + } + // Bits 8 through 21 are timestamp offset. + int timestamp_offset = + (payload_ptr[1] << 6) + ((payload_ptr[2] & 0xFC) >> 2); + new_header.timestamp = red_packet.timestamp - timestamp_offset; + // Bits 22 through 31 are payload length. + new_header.payload_length = + ((payload_ptr[2] & 0x03) << 8) + payload_ptr[3]; + + sum_length += new_header.payload_length; + sum_length += kRedHeaderLength; // Account for RED header size. + + payload_ptr += kRedHeaderLength; // Advance to next RED header. + payload_length -= kRedHeaderLength; + } + // Store in new list of packets. + if (new_header.payload_length > 0) { + new_headers.push_back(new_header); + } + } + + if (new_headers.size() <= kMaxRedBlocks) { + // Populate the new packets with payload data. + // `payload_ptr` now points at the first payload byte. + PacketList new_packets; // An empty list to store the split packets in. + for (size_t i = 0; i != new_headers.size(); ++i) { + const auto& new_header = new_headers[i]; + size_t payload_length = new_header.payload_length; + if (payload_ptr + payload_length > + red_packet.payload.data() + red_packet.payload.size()) { + // The block lengths in the RED headers do not match the overall + // packet length. Something is corrupt. Discard this and the remaining + // payloads from this packet. + RTC_LOG(LS_WARNING) << "SplitRed length mismatch"; + ret = false; + break; + } + + Packet new_packet; + new_packet.timestamp = new_header.timestamp; + new_packet.payload_type = new_header.payload_type; + new_packet.sequence_number = red_packet.sequence_number; + new_packet.priority.red_level = + rtc::dchecked_cast((new_headers.size() - 1) - i); + new_packet.payload.SetData(payload_ptr, payload_length); + new_packet.packet_info = RtpPacketInfo( + /*ssrc=*/red_packet.packet_info.ssrc(), + /*csrcs=*/std::vector(), + /*rtp_timestamp=*/new_packet.timestamp, + /*receive_time=*/red_packet.packet_info.receive_time()); + new_packet.packet_info.set_audio_level( + red_packet.packet_info.audio_level()); + new_packets.push_front(std::move(new_packet)); + payload_ptr += payload_length; + } + // Insert new packets into original list, before the element pointed to by + // iterator `it`. + packet_list->splice(it, std::move(new_packets)); + } else { + RTC_LOG(LS_WARNING) << "SplitRed too many blocks: " << new_headers.size(); + ret = false; + } + // Remove `it` from the packet list. This operation effectively moves the + // iterator `it` to the next packet in the list. Thus, we do not have to + // increment it manually. + it = packet_list->erase(it); + } + return ret; +} + +void RedPayloadSplitter::CheckRedPayloads( + PacketList* packet_list, + const DecoderDatabase& decoder_database) { + int main_payload_type = -1; + for (auto it = packet_list->begin(); it != packet_list->end(); /* */) { + uint8_t this_payload_type = it->payload_type; + if (decoder_database.IsRed(this_payload_type)) { + it = packet_list->erase(it); + continue; + } + if (!decoder_database.IsDtmf(this_payload_type) && + !decoder_database.IsComfortNoise(this_payload_type)) { + if (main_payload_type == -1) { + // This is the first packet in the list which is non-DTMF non-CNG. + main_payload_type = this_payload_type; + } else { + if (this_payload_type != main_payload_type) { + // We do not allow redundant payloads of a different type. + // Remove `it` from the packet list. This operation effectively + // moves the iterator `it` to the next packet in the list. Thus, we + // do not have to increment it manually. + it = packet_list->erase(it); + continue; + } + } + } + ++it; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.h b/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.h new file mode 100644 index 0000000000..2f48e4b7d4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ + +#include "modules/audio_coding/neteq/packet.h" + +namespace webrtc { + +class DecoderDatabase; + +static const size_t kRedHeaderLength = 4; // 4 bytes RED header. +static const size_t kRedLastHeaderLength = + 1; // reduced size for last RED header. +// This class handles splitting of RED payloads into smaller parts. +// Codec-specific packet splitting can be performed by +// AudioDecoder::ParsePayload. +class RedPayloadSplitter { + public: + RedPayloadSplitter() {} + + virtual ~RedPayloadSplitter() {} + + RedPayloadSplitter(const RedPayloadSplitter&) = delete; + RedPayloadSplitter& operator=(const RedPayloadSplitter&) = delete; + + // Splits each packet in `packet_list` into its separate RED payloads. Each + // RED payload is packetized into a Packet. The original elements in + // `packet_list` are properly deleted, and replaced by the new packets. + // Note that all packets in `packet_list` must be RED payloads, i.e., have + // RED headers according to RFC 2198 at the very beginning of the payload. + // Returns kOK or an error. + virtual bool SplitRed(PacketList* packet_list); + + // Checks all packets in `packet_list`. Packets that are DTMF events or + // comfort noise payloads are kept. Except that, only one single payload type + // is accepted. Any packet with another payload type is discarded. + virtual void CheckRedPayloads(PacketList* packet_list, + const DecoderDatabase& decoder_database); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_RED_PAYLOAD_SPLITTER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc new file mode 100644 index 0000000000..a0ba5414ea --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter_unittest.cc @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for RedPayloadSplitter class. + +#include "modules/audio_coding/neteq/red_payload_splitter.h" + + +#include +#include // pair + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "modules/audio_coding/neteq/packet.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" +#include "test/mock_audio_decoder_factory.h" + +using ::testing::Return; +using ::testing::ReturnNull; + +namespace webrtc { + +static const int kRedPayloadType = 100; +static const size_t kPayloadLength = 10; +static const uint16_t kSequenceNumber = 0; +static const uint32_t kBaseTimestamp = 0x12345678; + +// A possible Opus packet that contains FEC is the following. +// The frame is 20 ms in duration. +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |0|0|0|0|1|0|0|0|x|1|x|x|x|x|x|x|x| | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | +// | Compressed frame 1 (N-2 bytes)... : +// : | +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +void CreateOpusFecPayload(uint8_t* payload, + size_t payload_length, + uint8_t payload_value) { + if (payload_length < 2) { + return; + } + payload[0] = 0x08; + payload[1] = 0x40; + memset(&payload[2], payload_value, payload_length - 2); +} + +// RED headers (according to RFC 2198): +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |F| block PT | timestamp offset | block length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Last RED header: +// 0 1 2 3 4 5 6 7 +// +-+-+-+-+-+-+-+-+ +// |0| Block PT | +// +-+-+-+-+-+-+-+-+ + +// Creates a RED packet, with `num_payloads` payloads, with payload types given +// by the values in array `payload_types` (which must be of length +// `num_payloads`). Each redundant payload is `timestamp_offset` samples +// "behind" the the previous payload. +Packet CreateRedPayload(size_t num_payloads, + uint8_t* payload_types, + int timestamp_offset, + bool embed_opus_fec = false) { + Packet packet; + packet.payload_type = kRedPayloadType; + packet.timestamp = kBaseTimestamp; + packet.sequence_number = kSequenceNumber; + packet.payload.SetSize((kPayloadLength + 1) + + (num_payloads - 1) * + (kPayloadLength + kRedHeaderLength)); + uint8_t* payload_ptr = packet.payload.data(); + for (size_t i = 0; i < num_payloads; ++i) { + // Write the RED headers. + if (i == num_payloads - 1) { + // Special case for last payload. + *payload_ptr = payload_types[i] & 0x7F; // F = 0; + ++payload_ptr; + break; + } + *payload_ptr = payload_types[i] & 0x7F; + // Not the last block; set F = 1. + *payload_ptr |= 0x80; + ++payload_ptr; + int this_offset = + rtc::checked_cast((num_payloads - i - 1) * timestamp_offset); + *payload_ptr = this_offset >> 6; + ++payload_ptr; + RTC_DCHECK_LE(kPayloadLength, 1023); // Max length described by 10 bits. + *payload_ptr = ((this_offset & 0x3F) << 2) | (kPayloadLength >> 8); + ++payload_ptr; + *payload_ptr = kPayloadLength & 0xFF; + ++payload_ptr; + } + for (size_t i = 0; i < num_payloads; ++i) { + // Write `i` to all bytes in each payload. + if (embed_opus_fec) { + CreateOpusFecPayload(payload_ptr, kPayloadLength, + static_cast(i)); + } else { + memset(payload_ptr, static_cast(i), kPayloadLength); + } + payload_ptr += kPayloadLength; + } + return packet; +} + +// Create a packet with all payload bytes set to `payload_value`. +Packet CreatePacket(uint8_t payload_type, + size_t payload_length, + uint8_t payload_value, + bool opus_fec = false) { + Packet packet; + packet.payload_type = payload_type; + packet.timestamp = kBaseTimestamp; + packet.sequence_number = kSequenceNumber; + packet.payload.SetSize(payload_length); + if (opus_fec) { + CreateOpusFecPayload(packet.payload.data(), packet.payload.size(), + payload_value); + } else { + memset(packet.payload.data(), payload_value, packet.payload.size()); + } + return packet; +} + +// Checks that `packet` has the attributes given in the remaining parameters. +void VerifyPacket(const Packet& packet, + size_t payload_length, + uint8_t payload_type, + uint16_t sequence_number, + uint32_t timestamp, + uint8_t payload_value, + Packet::Priority priority) { + EXPECT_EQ(payload_length, packet.payload.size()); + EXPECT_EQ(payload_type, packet.payload_type); + EXPECT_EQ(sequence_number, packet.sequence_number); + EXPECT_EQ(timestamp, packet.timestamp); + EXPECT_EQ(priority, packet.priority); + ASSERT_FALSE(packet.payload.empty()); + for (size_t i = 0; i < packet.payload.size(); ++i) { + ASSERT_EQ(payload_value, packet.payload.data()[i]); + } +} + +void VerifyPacket(const Packet& packet, + size_t payload_length, + uint8_t payload_type, + uint16_t sequence_number, + uint32_t timestamp, + uint8_t payload_value, + bool primary) { + return VerifyPacket(packet, payload_length, payload_type, sequence_number, + timestamp, payload_value, + Packet::Priority{0, primary ? 0 : 1}); +} + +// Start of test definitions. + +TEST(RedPayloadSplitter, CreateAndDestroy) { + RedPayloadSplitter* splitter = new RedPayloadSplitter; + delete splitter; +} + +// Packet A is split into A1 and A2. +TEST(RedPayloadSplitter, OnePacketTwoPayloads) { + uint8_t payload_types[] = {0, 0}; + const int kTimestampOffset = 160; + PacketList packet_list; + packet_list.push_back(CreateRedPayload(2, payload_types, kTimestampOffset)); + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); + ASSERT_EQ(2u, packet_list.size()); + // Check first packet. The first in list should always be the primary payload. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[1], + kSequenceNumber, kBaseTimestamp, 1, true); + packet_list.pop_front(); + // Check second packet. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber, kBaseTimestamp - kTimestampOffset, 0, false); +} + +// Packets A and B are not split at all. Only the RED header in each packet is +// removed. +TEST(RedPayloadSplitter, TwoPacketsOnePayload) { + uint8_t payload_types[] = {0}; + const int kTimestampOffset = 160; + // Create first packet, with a single RED payload. + PacketList packet_list; + packet_list.push_back(CreateRedPayload(1, payload_types, kTimestampOffset)); + // Create second packet, with a single RED payload. + { + Packet packet = CreateRedPayload(1, payload_types, kTimestampOffset); + // Manually change timestamp and sequence number of second packet. + packet.timestamp += kTimestampOffset; + packet.sequence_number++; + packet_list.push_back(std::move(packet)); + } + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); + ASSERT_EQ(2u, packet_list.size()); + // Check first packet. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber, kBaseTimestamp, 0, true); + packet_list.pop_front(); + // Check second packet. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber + 1, kBaseTimestamp + kTimestampOffset, 0, true); +} + +// Packets A and B are split into packets A1, A2, A3, B1, B2, B3, with +// attributes as follows: +// +// A1* A2 A3 B1* B2 B3 +// Payload type 0 1 2 0 1 2 +// Timestamp b b-o b-2o b+o b b-o +// Sequence number 0 0 0 1 1 1 +// +// b = kBaseTimestamp, o = kTimestampOffset, * = primary. +TEST(RedPayloadSplitter, TwoPacketsThreePayloads) { + uint8_t payload_types[] = {2, 1, 0}; // Primary is the last one. + const int kTimestampOffset = 160; + // Create first packet, with 3 RED payloads. + PacketList packet_list; + packet_list.push_back(CreateRedPayload(3, payload_types, kTimestampOffset)); + // Create first packet, with 3 RED payloads. + { + Packet packet = CreateRedPayload(3, payload_types, kTimestampOffset); + // Manually change timestamp and sequence number of second packet. + packet.timestamp += kTimestampOffset; + packet.sequence_number++; + packet_list.push_back(std::move(packet)); + } + RedPayloadSplitter splitter; + EXPECT_TRUE(splitter.SplitRed(&packet_list)); + ASSERT_EQ(6u, packet_list.size()); + // Check first packet, A1. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[2], + kSequenceNumber, kBaseTimestamp, 2, {0, 0}); + packet_list.pop_front(); + // Check second packet, A2. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[1], + kSequenceNumber, kBaseTimestamp - kTimestampOffset, 1, {0, 1}); + packet_list.pop_front(); + // Check third packet, A3. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber, kBaseTimestamp - 2 * kTimestampOffset, 0, + {0, 2}); + packet_list.pop_front(); + // Check fourth packet, B1. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[2], + kSequenceNumber + 1, kBaseTimestamp + kTimestampOffset, 2, + {0, 0}); + packet_list.pop_front(); + // Check fifth packet, B2. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[1], + kSequenceNumber + 1, kBaseTimestamp, 1, {0, 1}); + packet_list.pop_front(); + // Check sixth packet, B3. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber + 1, kBaseTimestamp - kTimestampOffset, 0, + {0, 2}); +} + +// Creates a list with 4 packets with these payload types: +// 0 = CNGnb +// 1 = PCMu +// 2 = DTMF (AVT) +// 3 = iLBC +// We expect the method CheckRedPayloads to discard the iLBC packet, since it +// is a non-CNG, non-DTMF payload of another type than the first speech payload +// found in the list (which is PCMu). +TEST(RedPayloadSplitter, CheckRedPayloads) { + PacketList packet_list; + for (uint8_t i = 0; i <= 3; ++i) { + // Create packet with payload type `i`, payload length 10 bytes, all 0. + packet_list.push_back(CreatePacket(i, 10, 0)); + } + + // Use a real DecoderDatabase object here instead of a mock, since it is + // easier to just register the payload types and let the actual implementation + // do its job. + DecoderDatabase decoder_database( + rtc::make_ref_counted(), absl::nullopt); + decoder_database.RegisterPayload(0, SdpAudioFormat("cn", 8000, 1)); + decoder_database.RegisterPayload(1, SdpAudioFormat("pcmu", 8000, 1)); + decoder_database.RegisterPayload(2, + SdpAudioFormat("telephone-event", 8000, 1)); + decoder_database.RegisterPayload(3, SdpAudioFormat("ilbc", 8000, 1)); + + RedPayloadSplitter splitter; + splitter.CheckRedPayloads(&packet_list, decoder_database); + + ASSERT_EQ(3u, packet_list.size()); // Should have dropped the last packet. + // Verify packets. The loop verifies that payload types 0, 1, and 2 are in the + // list. + for (int i = 0; i <= 2; ++i) { + VerifyPacket(packet_list.front(), 10, i, kSequenceNumber, kBaseTimestamp, 0, + true); + packet_list.pop_front(); + } + EXPECT_TRUE(packet_list.empty()); +} + +// This test creates a RED packet where the payloads also have the payload type +// for RED. That is, some kind of weird nested RED packet. This is not supported +// and the splitter should discard all packets. +TEST(RedPayloadSplitter, CheckRedPayloadsRecursiveRed) { + PacketList packet_list; + for (uint8_t i = 0; i <= 3; ++i) { + // Create packet with RED payload type, payload length 10 bytes, all 0. + packet_list.push_back(CreatePacket(kRedPayloadType, 10, 0)); + } + + // Use a real DecoderDatabase object here instead of a mock, since it is + // easier to just register the payload types and let the actual implementation + // do its job. + DecoderDatabase decoder_database( + rtc::make_ref_counted(), absl::nullopt); + decoder_database.RegisterPayload(kRedPayloadType, + SdpAudioFormat("red", 8000, 1)); + + RedPayloadSplitter splitter; + splitter.CheckRedPayloads(&packet_list, decoder_database); + + EXPECT_TRUE(packet_list.empty()); // Should have dropped all packets. +} + +// Packet A is split into A1, A2 and A3. But the length parameter is off, so +// the last payloads should be discarded. +TEST(RedPayloadSplitter, WrongPayloadLength) { + uint8_t payload_types[] = {0, 0, 0}; + const int kTimestampOffset = 160; + PacketList packet_list; + { + Packet packet = CreateRedPayload(3, payload_types, kTimestampOffset); + // Manually tamper with the payload length of the packet. + // This is one byte too short for the second payload (out of three). + // We expect only the first payload to be returned. + packet.payload.SetSize(packet.payload.size() - (kPayloadLength + 1)); + packet_list.push_back(std::move(packet)); + } + RedPayloadSplitter splitter; + EXPECT_FALSE(splitter.SplitRed(&packet_list)); + ASSERT_EQ(1u, packet_list.size()); + // Check first packet. + VerifyPacket(packet_list.front(), kPayloadLength, payload_types[0], + kSequenceNumber, kBaseTimestamp - 2 * kTimestampOffset, 0, + {0, 2}); + packet_list.pop_front(); +} + +// Test that we reject packets too short to contain a RED header. +TEST(RedPayloadSplitter, RejectsIncompleteHeaders) { + RedPayloadSplitter splitter; + + uint8_t payload_types[] = {0, 0}; + const int kTimestampOffset = 160; + + PacketList packet_list; + + // Truncate the packet such that the first block can not be parsed. + packet_list.push_back(CreateRedPayload(2, payload_types, kTimestampOffset)); + packet_list.front().payload.SetSize(4); + EXPECT_FALSE(splitter.SplitRed(&packet_list)); + EXPECT_FALSE(packet_list.empty()); + + // Truncate the packet such that the first block can not be parsed. + packet_list.front().payload.SetSize(3); + EXPECT_FALSE(splitter.SplitRed(&packet_list)); + EXPECT_FALSE(packet_list.empty()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.cc b/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.cc new file mode 100644 index 0000000000..f6e073fc88 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/reorder_optimizer.h" + +#include +#include +#include + +namespace webrtc { + +namespace { + +constexpr int kDelayBuckets = 100; +constexpr int kBucketSizeMs = 20; + +} // namespace + +ReorderOptimizer::ReorderOptimizer(int forget_factor, + int ms_per_loss_percent, + absl::optional start_forget_weight) + : histogram_(kDelayBuckets, forget_factor, start_forget_weight), + ms_per_loss_percent_(ms_per_loss_percent) {} + +void ReorderOptimizer::Update(int relative_delay_ms, + bool reordered, + int base_delay_ms) { + const int index = reordered ? relative_delay_ms / kBucketSizeMs : 0; + if (index < histogram_.NumBuckets()) { + // Maximum delay to register is 2000 ms. + histogram_.Add(index); + } + int bucket_index = MinimizeCostFunction(base_delay_ms); + optimal_delay_ms_ = (1 + bucket_index) * kBucketSizeMs; +} + +void ReorderOptimizer::Reset() { + histogram_.Reset(); + optimal_delay_ms_.reset(); +} + +int ReorderOptimizer::MinimizeCostFunction(int base_delay_ms) const { + const std::vector& buckets = histogram_.buckets(); + + // Values are calculated in Q30. + int64_t loss_probability = 1 << 30; + int64_t min_cost = std::numeric_limits::max(); + int min_bucket = 0; + for (int i = 0; i < static_cast(buckets.size()); ++i) { + loss_probability -= buckets[i]; + int64_t delay_ms = + static_cast(std::max(0, i * kBucketSizeMs - base_delay_ms)) + << 30; + int64_t cost = delay_ms + 100 * ms_per_loss_percent_ * loss_probability; + + if (cost < min_cost) { + min_cost = cost; + min_bucket = i; + } + if (loss_probability == 0) { + break; + } + } + + return min_bucket; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.h b/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.h new file mode 100644 index 0000000000..06f6bc7e50 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_REORDER_OPTIMIZER_H_ +#define MODULES_AUDIO_CODING_NETEQ_REORDER_OPTIMIZER_H_ + +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/histogram.h" + +namespace webrtc { + +// Calculates an optimal delay to reduce the chance of missing reordered +// packets. The delay/loss trade-off can be tune using the `ms_per_loss_percent` +// parameter. +class ReorderOptimizer { + public: + ReorderOptimizer(int forget_factor, + int ms_per_loss_percent, + absl::optional start_forget_weight); + + void Update(int relative_delay_ms, bool reordered, int base_delay_ms); + + absl::optional GetOptimalDelayMs() const { return optimal_delay_ms_; } + + void Reset(); + + private: + int MinimizeCostFunction(int base_delay_ms) const; + + Histogram histogram_; + const int ms_per_loss_percent_; + absl::optional optimal_delay_ms_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_REORDER_OPTIMIZER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer_unittest.cc new file mode 100644 index 0000000000..aaa1062560 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer_unittest.cc @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/reorder_optimizer.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kForgetFactor = 32745; // 0.9993 in Q15. +constexpr int kMsPerLossPercent = 20; +constexpr int kStartForgetWeight = 1; + +} // namespace + +TEST(ReorderOptimizerTest, OnlyIncreaseDelayForReorderedPackets) { + ReorderOptimizer reorder_optimizer(kForgetFactor, kMsPerLossPercent, + kStartForgetWeight); + EXPECT_FALSE(reorder_optimizer.GetOptimalDelayMs()); + + // Delay should not increase for in-order packets. + reorder_optimizer.Update(60, /*reordered=*/false, 0); + EXPECT_EQ(reorder_optimizer.GetOptimalDelayMs(), 20); + + reorder_optimizer.Update(100, /*reordered=*/false, 0); + EXPECT_EQ(reorder_optimizer.GetOptimalDelayMs(), 20); + + reorder_optimizer.Update(80, /*reordered=*/true, 0); + EXPECT_EQ(reorder_optimizer.GetOptimalDelayMs(), 100); +} + +TEST(ReorderOptimizerTest, AvoidIncreasingDelayWhenProbabilityIsLow) { + ReorderOptimizer reorder_optimizer(kForgetFactor, kMsPerLossPercent, + kStartForgetWeight); + + reorder_optimizer.Update(40, /*reordered=*/true, 0); + reorder_optimizer.Update(40, /*reordered=*/true, 0); + reorder_optimizer.Update(40, /*reordered=*/true, 0); + EXPECT_EQ(reorder_optimizer.GetOptimalDelayMs(), 60); + + // The cost of the delay is too high relative the probability. + reorder_optimizer.Update(600, /*reordered=*/true, 0); + EXPECT_EQ(reorder_optimizer.GetOptimalDelayMs(), 60); +} + +TEST(ReorderOptimizerTest, BaseDelayIsSubtractedFromCost) { + constexpr int kBaseDelayMs = 200; + ReorderOptimizer reorder_optimizer(kForgetFactor, kMsPerLossPercent, + kStartForgetWeight); + + reorder_optimizer.Update(40, /*reordered=*/true, kBaseDelayMs); + reorder_optimizer.Update(40, /*reordered=*/true, kBaseDelayMs); + reorder_optimizer.Update(40, /*reordered=*/true, kBaseDelayMs); + EXPECT_EQ(reorder_optimizer.GetOptimalDelayMs(), 60); + + // The cost of the delay is too high relative the probability. + reorder_optimizer.Update(600, /*reordered=*/true, kBaseDelayMs); + EXPECT_EQ(reorder_optimizer.GetOptimalDelayMs(), 620); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.cc b/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.cc new file mode 100644 index 0000000000..52d3fa90f1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.cc @@ -0,0 +1,394 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/statistics_calculator.h" + +#include // memset + +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { +size_t AddIntToSizeTWithLowerCap(int a, size_t b) { + const size_t ret = b + a; + // If a + b is negative, resulting in a negative wrap, cap it to zero instead. + static_assert(sizeof(size_t) >= sizeof(int), + "int must not be wider than size_t for this to work"); + return (a < 0 && ret > b) ? 0 : ret; +} + +constexpr int kInterruptionLenMs = 150; +} // namespace + +// Allocating the static const so that it can be passed by reference to +// RTC_DCHECK. +const size_t StatisticsCalculator::kLenWaitingTimes; + +StatisticsCalculator::PeriodicUmaLogger::PeriodicUmaLogger( + absl::string_view uma_name, + int report_interval_ms, + int max_value) + : uma_name_(uma_name), + report_interval_ms_(report_interval_ms), + max_value_(max_value), + timer_(0) {} + +StatisticsCalculator::PeriodicUmaLogger::~PeriodicUmaLogger() = default; + +void StatisticsCalculator::PeriodicUmaLogger::AdvanceClock(int step_ms) { + timer_ += step_ms; + if (timer_ < report_interval_ms_) { + return; + } + LogToUma(Metric()); + Reset(); + timer_ -= report_interval_ms_; + RTC_DCHECK_GE(timer_, 0); +} + +void StatisticsCalculator::PeriodicUmaLogger::LogToUma(int value) const { + RTC_HISTOGRAM_COUNTS_SPARSE(uma_name_, value, 1, max_value_, 50); +} + +StatisticsCalculator::PeriodicUmaCount::PeriodicUmaCount( + absl::string_view uma_name, + int report_interval_ms, + int max_value) + : PeriodicUmaLogger(uma_name, report_interval_ms, max_value) {} + +StatisticsCalculator::PeriodicUmaCount::~PeriodicUmaCount() { + // Log the count for the current (incomplete) interval. + LogToUma(Metric()); +} + +void StatisticsCalculator::PeriodicUmaCount::RegisterSample() { + ++counter_; +} + +int StatisticsCalculator::PeriodicUmaCount::Metric() const { + return counter_; +} + +void StatisticsCalculator::PeriodicUmaCount::Reset() { + counter_ = 0; +} + +StatisticsCalculator::PeriodicUmaAverage::PeriodicUmaAverage( + absl::string_view uma_name, + int report_interval_ms, + int max_value) + : PeriodicUmaLogger(uma_name, report_interval_ms, max_value) {} + +StatisticsCalculator::PeriodicUmaAverage::~PeriodicUmaAverage() { + // Log the average for the current (incomplete) interval. + LogToUma(Metric()); +} + +void StatisticsCalculator::PeriodicUmaAverage::RegisterSample(int value) { + sum_ += value; + ++counter_; +} + +int StatisticsCalculator::PeriodicUmaAverage::Metric() const { + return counter_ == 0 ? 0 : static_cast(sum_ / counter_); +} + +void StatisticsCalculator::PeriodicUmaAverage::Reset() { + sum_ = 0.0; + counter_ = 0; +} + +StatisticsCalculator::StatisticsCalculator() + : preemptive_samples_(0), + accelerate_samples_(0), + expanded_speech_samples_(0), + expanded_noise_samples_(0), + timestamps_since_last_report_(0), + secondary_decoded_samples_(0), + discarded_secondary_packets_(0), + delayed_packet_outage_counter_( + "WebRTC.Audio.DelayedPacketOutageEventsPerMinute", + 60000, // 60 seconds report interval. + 100), + excess_buffer_delay_("WebRTC.Audio.AverageExcessBufferDelayMs", + 60000, // 60 seconds report interval. + 1000), + buffer_full_counter_("WebRTC.Audio.JitterBufferFullPerMinute", + 60000, // 60 seconds report interval. + 100) {} + +StatisticsCalculator::~StatisticsCalculator() = default; + +void StatisticsCalculator::Reset() { + preemptive_samples_ = 0; + accelerate_samples_ = 0; + expanded_speech_samples_ = 0; + expanded_noise_samples_ = 0; + secondary_decoded_samples_ = 0; + discarded_secondary_packets_ = 0; + waiting_times_.clear(); +} + +void StatisticsCalculator::ResetMcu() { + timestamps_since_last_report_ = 0; +} + +void StatisticsCalculator::ExpandedVoiceSamples(size_t num_samples, + bool is_new_concealment_event) { + expanded_speech_samples_ += num_samples; + ConcealedSamplesCorrection(rtc::dchecked_cast(num_samples), true); + lifetime_stats_.concealment_events += is_new_concealment_event; +} + +void StatisticsCalculator::ExpandedNoiseSamples(size_t num_samples, + bool is_new_concealment_event) { + expanded_noise_samples_ += num_samples; + ConcealedSamplesCorrection(rtc::dchecked_cast(num_samples), false); + lifetime_stats_.concealment_events += is_new_concealment_event; +} + +void StatisticsCalculator::ExpandedVoiceSamplesCorrection(int num_samples) { + expanded_speech_samples_ = + AddIntToSizeTWithLowerCap(num_samples, expanded_speech_samples_); + ConcealedSamplesCorrection(num_samples, true); +} + +void StatisticsCalculator::ExpandedNoiseSamplesCorrection(int num_samples) { + expanded_noise_samples_ = + AddIntToSizeTWithLowerCap(num_samples, expanded_noise_samples_); + ConcealedSamplesCorrection(num_samples, false); +} + +void StatisticsCalculator::DecodedOutputPlayed() { + decoded_output_played_ = true; +} + +void StatisticsCalculator::EndExpandEvent(int fs_hz) { + RTC_DCHECK_GE(lifetime_stats_.concealed_samples, + concealed_samples_at_event_end_); + const int event_duration_ms = + 1000 * + (lifetime_stats_.concealed_samples - concealed_samples_at_event_end_) / + fs_hz; + if (event_duration_ms >= kInterruptionLenMs && decoded_output_played_) { + lifetime_stats_.interruption_count++; + lifetime_stats_.total_interruption_duration_ms += event_duration_ms; + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AudioInterruptionMs", event_duration_ms, + /*min=*/150, /*max=*/5000, /*bucket_count=*/50); + } + concealed_samples_at_event_end_ = lifetime_stats_.concealed_samples; +} + +void StatisticsCalculator::ConcealedSamplesCorrection(int num_samples, + bool is_voice) { + if (num_samples < 0) { + // Store negative correction to subtract from future positive additions. + // See also the function comment in the header file. + concealed_samples_correction_ -= num_samples; + if (!is_voice) { + silent_concealed_samples_correction_ -= num_samples; + } + return; + } + + const size_t canceled_out = + std::min(static_cast(num_samples), concealed_samples_correction_); + concealed_samples_correction_ -= canceled_out; + lifetime_stats_.concealed_samples += num_samples - canceled_out; + + if (!is_voice) { + const size_t silent_canceled_out = std::min( + static_cast(num_samples), silent_concealed_samples_correction_); + silent_concealed_samples_correction_ -= silent_canceled_out; + lifetime_stats_.silent_concealed_samples += + num_samples - silent_canceled_out; + } +} + +void StatisticsCalculator::PreemptiveExpandedSamples(size_t num_samples) { + preemptive_samples_ += num_samples; + operations_and_state_.preemptive_samples += num_samples; + lifetime_stats_.inserted_samples_for_deceleration += num_samples; +} + +void StatisticsCalculator::AcceleratedSamples(size_t num_samples) { + accelerate_samples_ += num_samples; + operations_and_state_.accelerate_samples += num_samples; + lifetime_stats_.removed_samples_for_acceleration += num_samples; +} + +void StatisticsCalculator::GeneratedNoiseSamples(size_t num_samples) { + lifetime_stats_.generated_noise_samples += num_samples; +} + +void StatisticsCalculator::PacketsDiscarded(size_t num_packets) { + lifetime_stats_.packets_discarded += num_packets; +} + +void StatisticsCalculator::SecondaryPacketsDiscarded(size_t num_packets) { + discarded_secondary_packets_ += num_packets; + lifetime_stats_.fec_packets_discarded += num_packets; +} + +void StatisticsCalculator::SecondaryPacketsReceived(size_t num_packets) { + lifetime_stats_.fec_packets_received += num_packets; +} + +void StatisticsCalculator::IncreaseCounter(size_t num_samples, int fs_hz) { + const int time_step_ms = + rtc::CheckedDivExact(static_cast(1000 * num_samples), fs_hz); + delayed_packet_outage_counter_.AdvanceClock(time_step_ms); + excess_buffer_delay_.AdvanceClock(time_step_ms); + buffer_full_counter_.AdvanceClock(time_step_ms); + timestamps_since_last_report_ += static_cast(num_samples); + if (timestamps_since_last_report_ > + static_cast(fs_hz * kMaxReportPeriod)) { + timestamps_since_last_report_ = 0; + } + lifetime_stats_.total_samples_received += num_samples; +} + +void StatisticsCalculator::JitterBufferDelay( + size_t num_samples, + uint64_t waiting_time_ms, + uint64_t target_delay_ms, + uint64_t unlimited_target_delay_ms) { + lifetime_stats_.jitter_buffer_delay_ms += waiting_time_ms * num_samples; + lifetime_stats_.jitter_buffer_target_delay_ms += + target_delay_ms * num_samples; + lifetime_stats_.jitter_buffer_minimum_delay_ms += + unlimited_target_delay_ms * num_samples; + lifetime_stats_.jitter_buffer_emitted_count += num_samples; +} + +void StatisticsCalculator::SecondaryDecodedSamples(int num_samples) { + secondary_decoded_samples_ += num_samples; +} + +void StatisticsCalculator::FlushedPacketBuffer() { + operations_and_state_.packet_buffer_flushes++; + buffer_full_counter_.RegisterSample(); +} + +void StatisticsCalculator::ReceivedPacket() { + ++lifetime_stats_.jitter_buffer_packets_received; +} + +void StatisticsCalculator::RelativePacketArrivalDelay(size_t delay_ms) { + lifetime_stats_.relative_packet_arrival_delay_ms += delay_ms; +} + +void StatisticsCalculator::LogDelayedPacketOutageEvent(int num_samples, + int fs_hz) { + int outage_duration_ms = num_samples / (fs_hz / 1000); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.DelayedPacketOutageEventMs", + outage_duration_ms, 1 /* min */, 2000 /* max */, + 100 /* bucket count */); + delayed_packet_outage_counter_.RegisterSample(); + lifetime_stats_.delayed_packet_outage_samples += num_samples; +} + +void StatisticsCalculator::StoreWaitingTime(int waiting_time_ms) { + excess_buffer_delay_.RegisterSample(waiting_time_ms); + RTC_DCHECK_LE(waiting_times_.size(), kLenWaitingTimes); + if (waiting_times_.size() == kLenWaitingTimes) { + // Erase first value. + waiting_times_.pop_front(); + } + waiting_times_.push_back(waiting_time_ms); + operations_and_state_.last_waiting_time_ms = waiting_time_ms; +} + +void StatisticsCalculator::GetNetworkStatistics(size_t samples_per_packet, + NetEqNetworkStatistics* stats) { + RTC_DCHECK(stats); + + stats->accelerate_rate = + CalculateQ14Ratio(accelerate_samples_, timestamps_since_last_report_); + + stats->preemptive_rate = + CalculateQ14Ratio(preemptive_samples_, timestamps_since_last_report_); + + stats->expand_rate = + CalculateQ14Ratio(expanded_speech_samples_ + expanded_noise_samples_, + timestamps_since_last_report_); + + stats->speech_expand_rate = CalculateQ14Ratio(expanded_speech_samples_, + timestamps_since_last_report_); + + stats->secondary_decoded_rate = CalculateQ14Ratio( + secondary_decoded_samples_, timestamps_since_last_report_); + + const size_t discarded_secondary_samples = + discarded_secondary_packets_ * samples_per_packet; + stats->secondary_discarded_rate = + CalculateQ14Ratio(discarded_secondary_samples, + static_cast(discarded_secondary_samples + + secondary_decoded_samples_)); + + if (waiting_times_.size() == 0) { + stats->mean_waiting_time_ms = -1; + stats->median_waiting_time_ms = -1; + stats->min_waiting_time_ms = -1; + stats->max_waiting_time_ms = -1; + } else { + std::sort(waiting_times_.begin(), waiting_times_.end()); + // Find mid-point elements. If the size is odd, the two values + // `middle_left` and `middle_right` will both be the one middle element; if + // the size is even, they will be the the two neighboring elements at the + // middle of the list. + const int middle_left = waiting_times_[(waiting_times_.size() - 1) / 2]; + const int middle_right = waiting_times_[waiting_times_.size() / 2]; + // Calculate the average of the two. (Works also for odd sizes.) + stats->median_waiting_time_ms = (middle_left + middle_right) / 2; + stats->min_waiting_time_ms = waiting_times_.front(); + stats->max_waiting_time_ms = waiting_times_.back(); + double sum = 0; + for (auto time : waiting_times_) { + sum += time; + } + stats->mean_waiting_time_ms = static_cast(sum / waiting_times_.size()); + } + + // Reset counters. + ResetMcu(); + Reset(); +} + +NetEqLifetimeStatistics StatisticsCalculator::GetLifetimeStatistics() const { + return lifetime_stats_; +} + +NetEqOperationsAndState StatisticsCalculator::GetOperationsAndState() const { + return operations_and_state_; +} + +uint16_t StatisticsCalculator::CalculateQ14Ratio(size_t numerator, + uint32_t denominator) { + if (numerator == 0) { + return 0; + } else if (numerator < denominator) { + // Ratio must be smaller than 1 in Q14. + RTC_DCHECK_LT((numerator << 14) / denominator, (1 << 14)); + return static_cast((numerator << 14) / denominator); + } else { + // Will not produce a ratio larger than 1, since this is probably an error. + return 1 << 14; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.h b/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.h new file mode 100644 index 0000000000..33a22d02dd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.h @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_STATISTICS_CALCULATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_STATISTICS_CALCULATOR_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "api/neteq/neteq.h" + +namespace webrtc { + +class DelayManager; + +// This class handles various network statistics in NetEq. +class StatisticsCalculator { + public: + StatisticsCalculator(); + + virtual ~StatisticsCalculator(); + + StatisticsCalculator(const StatisticsCalculator&) = delete; + StatisticsCalculator& operator=(const StatisticsCalculator&) = delete; + + // Resets most of the counters. + void Reset(); + + // Resets the counters that are not handled by Reset(). + void ResetMcu(); + + // Reports that `num_samples` samples were produced through expansion, and + // that the expansion produced other than just noise samples. + void ExpandedVoiceSamples(size_t num_samples, bool is_new_concealment_event); + + // Reports that `num_samples` samples were produced through expansion, and + // that the expansion produced only noise samples. + void ExpandedNoiseSamples(size_t num_samples, bool is_new_concealment_event); + + // Corrects the statistics for number of samples produced through non-noise + // expansion by adding `num_samples` (negative or positive) to the current + // value. The result is capped to zero to avoid negative values. + void ExpandedVoiceSamplesCorrection(int num_samples); + + // Same as ExpandedVoiceSamplesCorrection but for noise samples. + void ExpandedNoiseSamplesCorrection(int num_samples); + + void DecodedOutputPlayed(); + + // Mark end of expand event; triggers some stats to be reported. + void EndExpandEvent(int fs_hz); + + // Reports that `num_samples` samples were produced through preemptive + // expansion. + void PreemptiveExpandedSamples(size_t num_samples); + + // Reports that `num_samples` samples were removed through accelerate. + void AcceleratedSamples(size_t num_samples); + + // Reports that `num_samples` comfort noise samples were generated. + void GeneratedNoiseSamples(size_t num_samples); + + // Reports that `num_packets` packets were discarded. + virtual void PacketsDiscarded(size_t num_packets); + + // Reports that `num_packets` secondary (FEC) packets were discarded. + virtual void SecondaryPacketsDiscarded(size_t num_packets); + + // Reports that `num_packets` secondary (FEC) packets were received. + virtual void SecondaryPacketsReceived(size_t num_packets); + + // Increases the report interval counter with `num_samples` at a sample rate + // of `fs_hz`. This is how the StatisticsCalculator gets notified that current + // time is increasing. + void IncreaseCounter(size_t num_samples, int fs_hz); + + // Update jitter buffer delay counter. + void JitterBufferDelay(size_t num_samples, + uint64_t waiting_time_ms, + uint64_t target_delay_ms, + uint64_t unlimited_target_delay_ms); + + // Stores new packet waiting time in waiting time statistics. + void StoreWaitingTime(int waiting_time_ms); + + // Reports that `num_samples` samples were decoded from secondary packets. + void SecondaryDecodedSamples(int num_samples); + + // Reports that the packet buffer was flushed. + void FlushedPacketBuffer(); + + // Reports that the jitter buffer received a packet. + void ReceivedPacket(); + + // Reports that a received packet was delayed by `delay_ms` milliseconds. + virtual void RelativePacketArrivalDelay(size_t delay_ms); + + // Logs a delayed packet outage event of `num_samples` expanded at a sample + // rate of `fs_hz`. A delayed packet outage event is defined as an expand + // period caused not by an actual packet loss, but by a delayed packet. + virtual void LogDelayedPacketOutageEvent(int num_samples, int fs_hz); + + // Returns the current network statistics in `stats`. The number of samples + // per packet is `samples_per_packet`. The method does not populate + // `preferred_buffer_size_ms`, `jitter_peaks_found` or `clockdrift_ppm`; use + // the PopulateDelayManagerStats method for those. + void GetNetworkStatistics(size_t samples_per_packet, + NetEqNetworkStatistics* stats); + + // Returns a copy of this class's lifetime statistics. These statistics are + // never reset. + NetEqLifetimeStatistics GetLifetimeStatistics() const; + + NetEqOperationsAndState GetOperationsAndState() const; + + private: + static const int kMaxReportPeriod = 60; // Seconds before auto-reset. + static const size_t kLenWaitingTimes = 100; + + class PeriodicUmaLogger { + public: + PeriodicUmaLogger(absl::string_view uma_name, + int report_interval_ms, + int max_value); + virtual ~PeriodicUmaLogger(); + void AdvanceClock(int step_ms); + + protected: + void LogToUma(int value) const; + virtual int Metric() const = 0; + virtual void Reset() = 0; + + const std::string uma_name_; + const int report_interval_ms_; + const int max_value_; + int timer_ = 0; + }; + + class PeriodicUmaCount final : public PeriodicUmaLogger { + public: + PeriodicUmaCount(absl::string_view uma_name, + int report_interval_ms, + int max_value); + ~PeriodicUmaCount() override; + void RegisterSample(); + + protected: + int Metric() const override; + void Reset() override; + + private: + int counter_ = 0; + }; + + class PeriodicUmaAverage final : public PeriodicUmaLogger { + public: + PeriodicUmaAverage(absl::string_view uma_name, + int report_interval_ms, + int max_value); + ~PeriodicUmaAverage() override; + void RegisterSample(int value); + + protected: + int Metric() const override; + void Reset() override; + + private: + double sum_ = 0.0; + int counter_ = 0; + }; + + // Corrects the concealed samples counter in lifetime_stats_. The value of + // num_samples_ is added directly to the stat if the correction is positive. + // If the correction is negative, it is cached and will be subtracted against + // future additions to the counter. This is meant to be called from + // Expanded{Voice,Noise}Samples{Correction}. + void ConcealedSamplesCorrection(int num_samples, bool is_voice); + + // Calculates numerator / denominator, and returns the value in Q14. + static uint16_t CalculateQ14Ratio(size_t numerator, uint32_t denominator); + + NetEqLifetimeStatistics lifetime_stats_; + NetEqOperationsAndState operations_and_state_; + size_t concealed_samples_correction_ = 0; + size_t silent_concealed_samples_correction_ = 0; + size_t preemptive_samples_; + size_t accelerate_samples_; + size_t expanded_speech_samples_; + size_t expanded_noise_samples_; + size_t concealed_samples_at_event_end_ = 0; + uint32_t timestamps_since_last_report_; + std::deque waiting_times_; + uint32_t secondary_decoded_samples_; + size_t discarded_secondary_packets_; + PeriodicUmaCount delayed_packet_outage_counter_; + PeriodicUmaAverage excess_buffer_delay_; + PeriodicUmaCount buffer_full_counter_; + bool decoded_output_played_ = false; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_STATISTICS_CALCULATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator_unittest.cc new file mode 100644 index 0000000000..491cd83dc4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator_unittest.cc @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/statistics_calculator.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(LifetimeStatistics, TotalSamplesReceived) { + StatisticsCalculator stats; + for (int i = 0; i < 10; ++i) { + stats.IncreaseCounter(480, 48000); // 10 ms at 48 kHz. + } + EXPECT_EQ(10 * 480u, stats.GetLifetimeStatistics().total_samples_received); +} + +TEST(LifetimeStatistics, SamplesConcealed) { + StatisticsCalculator stats; + stats.ExpandedVoiceSamples(100, false); + stats.ExpandedNoiseSamples(17, false); + EXPECT_EQ(100u + 17u, stats.GetLifetimeStatistics().concealed_samples); +} + +// This test verifies that a negative correction of concealed_samples does not +// result in a decrease in the stats value (because stats-consuming applications +// would not expect the value to decrease). Instead, the correction should be +// made to future increments to the stat. +TEST(LifetimeStatistics, SamplesConcealedCorrection) { + StatisticsCalculator stats; + stats.ExpandedVoiceSamples(100, false); + EXPECT_EQ(100u, stats.GetLifetimeStatistics().concealed_samples); + stats.ExpandedVoiceSamplesCorrection(-10); + // Do not subtract directly, but keep the correction for later. + EXPECT_EQ(100u, stats.GetLifetimeStatistics().concealed_samples); + stats.ExpandedVoiceSamplesCorrection(20); + // The total correction is 20 - 10. + EXPECT_EQ(110u, stats.GetLifetimeStatistics().concealed_samples); + + // Also test correction done to the next ExpandedVoiceSamples call. + stats.ExpandedVoiceSamplesCorrection(-17); + EXPECT_EQ(110u, stats.GetLifetimeStatistics().concealed_samples); + stats.ExpandedVoiceSamples(100, false); + EXPECT_EQ(110u + 100u - 17u, stats.GetLifetimeStatistics().concealed_samples); +} + +// This test verifies that neither "accelerate" nor "pre-emptive expand" reults +// in a modification to concealed_samples stats. Only PLC operations (i.e., +// "expand" and "merge") should affect the stat. +TEST(LifetimeStatistics, NoUpdateOnTimeStretch) { + StatisticsCalculator stats; + stats.ExpandedVoiceSamples(100, false); + stats.AcceleratedSamples(4711); + stats.PreemptiveExpandedSamples(17); + stats.ExpandedVoiceSamples(100, false); + EXPECT_EQ(200u, stats.GetLifetimeStatistics().concealed_samples); +} + +TEST(StatisticsCalculator, ExpandedSamplesCorrection) { + StatisticsCalculator stats; + NetEqNetworkStatistics stats_output; + constexpr int kSampleRateHz = 48000; + constexpr int k10MsSamples = kSampleRateHz / 100; + constexpr int kPacketSizeMs = 20; + constexpr size_t kSamplesPerPacket = kPacketSizeMs * kSampleRateHz / 1000; + + // Advance time by 10 ms. + stats.IncreaseCounter(k10MsSamples, kSampleRateHz); + + stats.GetNetworkStatistics(kSamplesPerPacket, &stats_output); + + EXPECT_EQ(0u, stats_output.expand_rate); + EXPECT_EQ(0u, stats_output.speech_expand_rate); + + // Correct with a negative value. + stats.ExpandedVoiceSamplesCorrection(-100); + stats.ExpandedNoiseSamplesCorrection(-100); + stats.IncreaseCounter(k10MsSamples, kSampleRateHz); + stats.GetNetworkStatistics(kSamplesPerPacket, &stats_output); + // Expect no change, since negative values are disallowed. + EXPECT_EQ(0u, stats_output.expand_rate); + EXPECT_EQ(0u, stats_output.speech_expand_rate); + + // Correct with a positive value. + stats.ExpandedVoiceSamplesCorrection(50); + stats.ExpandedNoiseSamplesCorrection(200); + stats.IncreaseCounter(k10MsSamples, kSampleRateHz); + stats.GetNetworkStatistics(kSamplesPerPacket, &stats_output); + // Calculate expected rates in Q14. Expand rate is noise + voice, while + // speech expand rate is only voice. + EXPECT_EQ(((50u + 200u) << 14) / k10MsSamples, stats_output.expand_rate); + EXPECT_EQ((50u << 14) / k10MsSamples, stats_output.speech_expand_rate); +} + +TEST(StatisticsCalculator, RelativePacketArrivalDelay) { + StatisticsCalculator stats; + + stats.RelativePacketArrivalDelay(50); + NetEqLifetimeStatistics stats_output = stats.GetLifetimeStatistics(); + EXPECT_EQ(50u, stats_output.relative_packet_arrival_delay_ms); + + stats.RelativePacketArrivalDelay(20); + stats_output = stats.GetLifetimeStatistics(); + EXPECT_EQ(70u, stats_output.relative_packet_arrival_delay_ms); +} + +TEST(StatisticsCalculator, ReceivedPacket) { + StatisticsCalculator stats; + + stats.ReceivedPacket(); + NetEqLifetimeStatistics stats_output = stats.GetLifetimeStatistics(); + EXPECT_EQ(1u, stats_output.jitter_buffer_packets_received); + + stats.ReceivedPacket(); + stats_output = stats.GetLifetimeStatistics(); + EXPECT_EQ(2u, stats_output.jitter_buffer_packets_received); +} + +TEST(StatisticsCalculator, InterruptionCounter) { + constexpr int fs_khz = 48; + constexpr int fs_hz = fs_khz * 1000; + StatisticsCalculator stats; + stats.DecodedOutputPlayed(); + stats.EndExpandEvent(fs_hz); + auto lts = stats.GetLifetimeStatistics(); + EXPECT_EQ(0, lts.interruption_count); + EXPECT_EQ(0, lts.total_interruption_duration_ms); + + // Add an event that is shorter than 150 ms. Should not be logged. + stats.ExpandedVoiceSamples(10 * fs_khz, false); // 10 ms. + stats.ExpandedNoiseSamples(139 * fs_khz, false); // 139 ms. + stats.EndExpandEvent(fs_hz); + lts = stats.GetLifetimeStatistics(); + EXPECT_EQ(0, lts.interruption_count); + + // Add an event that is longer than 150 ms. Should be logged. + stats.ExpandedVoiceSamples(140 * fs_khz, false); // 140 ms. + stats.ExpandedNoiseSamples(11 * fs_khz, false); // 11 ms. + stats.EndExpandEvent(fs_hz); + lts = stats.GetLifetimeStatistics(); + EXPECT_EQ(1, lts.interruption_count); + EXPECT_EQ(151, lts.total_interruption_duration_ms); + + // Add one more long event. + stats.ExpandedVoiceSamples(100 * fs_khz, false); // 100 ms. + stats.ExpandedNoiseSamples(5000 * fs_khz, false); // 5000 ms. + stats.EndExpandEvent(fs_hz); + lts = stats.GetLifetimeStatistics(); + EXPECT_EQ(2, lts.interruption_count); + EXPECT_EQ(5100 + 151, lts.total_interruption_duration_ms); +} + +TEST(StatisticsCalculator, InterruptionCounterDoNotLogBeforeDecoding) { + constexpr int fs_khz = 48; + constexpr int fs_hz = fs_khz * 1000; + StatisticsCalculator stats; + + // Add an event that is longer than 150 ms. Should normally be logged, but we + // have not called DecodedOutputPlayed() yet, so it shouldn't this time. + stats.ExpandedVoiceSamples(151 * fs_khz, false); // 151 ms. + stats.EndExpandEvent(fs_hz); + auto lts = stats.GetLifetimeStatistics(); + EXPECT_EQ(0, lts.interruption_count); + + // Call DecodedOutputPlayed(). Logging should happen after this. + stats.DecodedOutputPlayed(); + + // Add one more long event. + stats.ExpandedVoiceSamples(151 * fs_khz, false); // 151 ms. + stats.EndExpandEvent(fs_hz); + lts = stats.GetLifetimeStatistics(); + EXPECT_EQ(1, lts.interruption_count); +} + +TEST(StatisticsCalculator, DiscardedPackets) { + StatisticsCalculator statistics_calculator; + EXPECT_EQ(0u, + statistics_calculator.GetLifetimeStatistics().packets_discarded); + + statistics_calculator.PacketsDiscarded(1); + EXPECT_EQ(1u, + statistics_calculator.GetLifetimeStatistics().packets_discarded); + + statistics_calculator.PacketsDiscarded(10); + EXPECT_EQ(11u, + statistics_calculator.GetLifetimeStatistics().packets_discarded); + + // Calling `SecondaryPacketsDiscarded` does not modify `packets_discarded`. + statistics_calculator.SecondaryPacketsDiscarded(1); + EXPECT_EQ(11u, + statistics_calculator.GetLifetimeStatistics().packets_discarded); + + // Calling `FlushedPacketBuffer` does not modify `packets_discarded`. + statistics_calculator.FlushedPacketBuffer(); + EXPECT_EQ(11u, + statistics_calculator.GetLifetimeStatistics().packets_discarded); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.cc b/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.cc new file mode 100644 index 0000000000..7d7cac7157 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/sync_buffer.h" + +#include // Access to min. + +#include "rtc_base/checks.h" + +namespace webrtc { + +size_t SyncBuffer::FutureLength() const { + return Size() - next_index_; +} + +void SyncBuffer::PushBack(const AudioMultiVector& append_this) { + size_t samples_added = append_this.Size(); + AudioMultiVector::PushBack(append_this); + AudioMultiVector::PopFront(samples_added); + if (samples_added <= next_index_) { + next_index_ -= samples_added; + } else { + // This means that we are pushing out future data that was never used. + // RTC_DCHECK_NOTREACHED(); + // TODO(hlundin): This assert must be disabled to support 60 ms frames. + // This should not happen even for 60 ms frames, but it does. Investigate + // why. + next_index_ = 0; + } + dtmf_index_ -= std::min(dtmf_index_, samples_added); +} + +void SyncBuffer::PushBackInterleaved(const rtc::BufferT& append_this) { + const size_t size_before_adding = Size(); + AudioMultiVector::PushBackInterleaved(append_this); + const size_t samples_added_per_channel = Size() - size_before_adding; + RTC_DCHECK_EQ(samples_added_per_channel * Channels(), append_this.size()); + AudioMultiVector::PopFront(samples_added_per_channel); + next_index_ -= std::min(next_index_, samples_added_per_channel); + dtmf_index_ -= std::min(dtmf_index_, samples_added_per_channel); +} + +void SyncBuffer::PushFrontZeros(size_t length) { + InsertZerosAtIndex(length, 0); +} + +void SyncBuffer::InsertZerosAtIndex(size_t length, size_t position) { + position = std::min(position, Size()); + length = std::min(length, Size() - position); + AudioMultiVector::PopBack(length); + for (size_t channel = 0; channel < Channels(); ++channel) { + channels_[channel]->InsertZerosAt(length, position); + } + if (next_index_ >= position) { + // We are moving the `next_index_` sample. + set_next_index(next_index_ + length); // Overflow handled by subfunction. + } + if (dtmf_index_ > 0 && dtmf_index_ >= position) { + // We are moving the `dtmf_index_` sample. + set_dtmf_index(dtmf_index_ + length); // Overflow handled by subfunction. + } +} + +void SyncBuffer::ReplaceAtIndex(const AudioMultiVector& insert_this, + size_t length, + size_t position) { + position = std::min(position, Size()); // Cap `position` in the valid range. + length = std::min(length, Size() - position); + AudioMultiVector::OverwriteAt(insert_this, length, position); +} + +void SyncBuffer::ReplaceAtIndex(const AudioMultiVector& insert_this, + size_t position) { + ReplaceAtIndex(insert_this, insert_this.Size(), position); +} + +void SyncBuffer::GetNextAudioInterleaved(size_t requested_len, + AudioFrame* output) { + RTC_DCHECK(output); + const size_t samples_to_read = std::min(FutureLength(), requested_len); + output->ResetWithoutMuting(); + const size_t tot_samples_read = ReadInterleavedFromIndex( + next_index_, samples_to_read, output->mutable_data()); + const size_t samples_read_per_channel = tot_samples_read / Channels(); + next_index_ += samples_read_per_channel; + output->num_channels_ = Channels(); + output->samples_per_channel_ = samples_read_per_channel; +} + +void SyncBuffer::IncreaseEndTimestamp(uint32_t increment) { + end_timestamp_ += increment; +} + +void SyncBuffer::Flush() { + Zeros(Size()); + next_index_ = Size(); + end_timestamp_ = 0; + dtmf_index_ = 0; +} + +void SyncBuffer::set_next_index(size_t value) { + // Cannot set `next_index_` larger than the size of the buffer. + next_index_ = std::min(value, Size()); +} + +void SyncBuffer::set_dtmf_index(size_t value) { + // Cannot set `dtmf_index_` larger than the size of the buffer. + dtmf_index_ = std::min(value, Size()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.h b/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.h new file mode 100644 index 0000000000..cf56c432e3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_SYNC_BUFFER_H_ +#define MODULES_AUDIO_CODING_NETEQ_SYNC_BUFFER_H_ + +#include +#include + +#include + +#include "api/audio/audio_frame.h" +#include "modules/audio_coding/neteq/audio_multi_vector.h" +#include "modules/audio_coding/neteq/audio_vector.h" +#include "rtc_base/buffer.h" + +namespace webrtc { + +class SyncBuffer : public AudioMultiVector { + public: + SyncBuffer(size_t channels, size_t length) + : AudioMultiVector(channels, length), + next_index_(length), + end_timestamp_(0), + dtmf_index_(0) {} + + SyncBuffer(const SyncBuffer&) = delete; + SyncBuffer& operator=(const SyncBuffer&) = delete; + + // Returns the number of samples yet to play out from the buffer. + size_t FutureLength() const; + + // Adds the contents of `append_this` to the back of the SyncBuffer. Removes + // the same number of samples from the beginning of the SyncBuffer, to + // maintain a constant buffer size. The `next_index_` is updated to reflect + // the move of the beginning of "future" data. + void PushBack(const AudioMultiVector& append_this) override; + + // Like PushBack, but reads the samples channel-interleaved from the input. + void PushBackInterleaved(const rtc::BufferT& append_this); + + // Adds `length` zeros to the beginning of each channel. Removes + // the same number of samples from the end of the SyncBuffer, to + // maintain a constant buffer size. The `next_index_` is updated to reflect + // the move of the beginning of "future" data. + // Note that this operation may delete future samples that are waiting to + // be played. + void PushFrontZeros(size_t length); + + // Inserts `length` zeros into each channel at index `position`. The size of + // the SyncBuffer is kept constant, which means that the last `length` + // elements in each channel will be purged. + virtual void InsertZerosAtIndex(size_t length, size_t position); + + // Overwrites each channel in this SyncBuffer with values taken from + // `insert_this`. The values are taken from the beginning of `insert_this` and + // are inserted starting at `position`. `length` values are written into each + // channel. The size of the SyncBuffer is kept constant. That is, if `length` + // and `position` are selected such that the new data would extend beyond the + // end of the current SyncBuffer, the buffer is not extended. + // The `next_index_` is not updated. + virtual void ReplaceAtIndex(const AudioMultiVector& insert_this, + size_t length, + size_t position); + + // Same as the above method, but where all of `insert_this` is written (with + // the same constraints as above, that the SyncBuffer is not extended). + virtual void ReplaceAtIndex(const AudioMultiVector& insert_this, + size_t position); + + // Reads `requested_len` samples from each channel and writes them interleaved + // into `output`. The `next_index_` is updated to point to the sample to read + // next time. The AudioFrame `output` is first reset, and the `data_`, + // `num_channels_`, and `samples_per_channel_` fields are updated. + void GetNextAudioInterleaved(size_t requested_len, AudioFrame* output); + + // Adds `increment` to `end_timestamp_`. + void IncreaseEndTimestamp(uint32_t increment); + + // Flushes the buffer. The buffer will contain only zeros after the flush, and + // `next_index_` will point to the end, like when the buffer was first + // created. + void Flush(); + + const AudioVector& Channel(size_t n) const { return *channels_[n]; } + AudioVector& Channel(size_t n) { return *channels_[n]; } + + // Accessors and mutators. + size_t next_index() const { return next_index_; } + void set_next_index(size_t value); + uint32_t end_timestamp() const { return end_timestamp_; } + void set_end_timestamp(uint32_t value) { end_timestamp_ = value; } + size_t dtmf_index() const { return dtmf_index_; } + void set_dtmf_index(size_t value); + + private: + size_t next_index_; + uint32_t end_timestamp_; // The timestamp of the last sample in the buffer. + size_t dtmf_index_; // Index to the first non-DTMF sample in the buffer. +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_SYNC_BUFFER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer_unittest.cc new file mode 100644 index 0000000000..bdcd92446b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer_unittest.cc @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/sync_buffer.h" + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(SyncBuffer, CreateAndDestroy) { + // Create a SyncBuffer with two channels and 10 samples each. + static const size_t kLen = 10; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + EXPECT_EQ(kChannels, sync_buffer.Channels()); + EXPECT_EQ(kLen, sync_buffer.Size()); + // When the buffer is empty, the next index to play out is at the end. + EXPECT_EQ(kLen, sync_buffer.next_index()); + // Verify that all elements are zero. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kLen; ++i) { + EXPECT_EQ(0, sync_buffer[channel][i]); + } + } +} + +TEST(SyncBuffer, SetNextIndex) { + // Create a SyncBuffer with two channels and 100 samples each. + static const size_t kLen = 100; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + sync_buffer.set_next_index(0); + EXPECT_EQ(0u, sync_buffer.next_index()); + sync_buffer.set_next_index(kLen / 2); + EXPECT_EQ(kLen / 2, sync_buffer.next_index()); + sync_buffer.set_next_index(kLen); + EXPECT_EQ(kLen, sync_buffer.next_index()); + // Try to set larger than the buffer size; should cap at buffer size. + sync_buffer.set_next_index(kLen + 1); + EXPECT_EQ(kLen, sync_buffer.next_index()); +} + +TEST(SyncBuffer, PushBackAndFlush) { + // Create a SyncBuffer with two channels and 100 samples each. + static const size_t kLen = 100; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + static const size_t kNewLen = 10; + AudioMultiVector new_data(kChannels, kNewLen); + // Populate `new_data`. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen; ++i) { + new_data[channel][i] = rtc::checked_cast(i); + } + } + // Push back `new_data` into `sync_buffer`. This operation should pop out + // data from the front of `sync_buffer`, so that the size of the buffer + // remains the same. The `next_index_` should also move with the same length. + sync_buffer.PushBack(new_data); + ASSERT_EQ(kLen, sync_buffer.Size()); + // Verify that `next_index_` moved accordingly. + EXPECT_EQ(kLen - kNewLen, sync_buffer.next_index()); + // Verify the new contents. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen; ++i) { + EXPECT_EQ(new_data[channel][i], + sync_buffer[channel][sync_buffer.next_index() + i]); + } + } + + // Now flush the buffer, and verify that it is all zeros, and that next_index + // points to the end. + sync_buffer.Flush(); + ASSERT_EQ(kLen, sync_buffer.Size()); + EXPECT_EQ(kLen, sync_buffer.next_index()); + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kLen; ++i) { + EXPECT_EQ(0, sync_buffer[channel][i]); + } + } +} + +TEST(SyncBuffer, PushFrontZeros) { + // Create a SyncBuffer with two channels and 100 samples each. + static const size_t kLen = 100; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + static const size_t kNewLen = 10; + AudioMultiVector new_data(kChannels, kNewLen); + // Populate `new_data`. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen; ++i) { + new_data[channel][i] = rtc::checked_cast(1000 + i); + } + } + sync_buffer.PushBack(new_data); + EXPECT_EQ(kLen, sync_buffer.Size()); + + // Push `kNewLen` - 1 zeros into each channel in the front of the SyncBuffer. + sync_buffer.PushFrontZeros(kNewLen - 1); + EXPECT_EQ(kLen, sync_buffer.Size()); // Size should remain the same. + // Verify that `next_index_` moved accordingly. Should be at the end - 1. + EXPECT_EQ(kLen - 1, sync_buffer.next_index()); + // Verify the zeros. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen - 1; ++i) { + EXPECT_EQ(0, sync_buffer[channel][i]); + } + } + // Verify that the correct data is at the end of the SyncBuffer. + for (size_t channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(1000, sync_buffer[channel][sync_buffer.next_index()]); + } +} + +TEST(SyncBuffer, GetNextAudioInterleaved) { + // Create a SyncBuffer with two channels and 100 samples each. + static const size_t kLen = 100; + static const size_t kChannels = 2; + SyncBuffer sync_buffer(kChannels, kLen); + static const size_t kNewLen = 10; + AudioMultiVector new_data(kChannels, kNewLen); + // Populate `new_data`. + for (size_t channel = 0; channel < kChannels; ++channel) { + for (size_t i = 0; i < kNewLen; ++i) { + new_data[channel][i] = rtc::checked_cast(i); + } + } + // Push back `new_data` into `sync_buffer`. This operation should pop out + // data from the front of `sync_buffer`, so that the size of the buffer + // remains the same. The `next_index_` should also move with the same length. + sync_buffer.PushBack(new_data); + + // Read to interleaved output. Read in two batches, where each read operation + // should automatically update the `net_index_` in the SyncBuffer. + // Note that `samples_read` is the number of samples read from each channel. + // That is, the number of samples written to `output` is + // `samples_read` * `kChannels`. + AudioFrame output1; + sync_buffer.GetNextAudioInterleaved(kNewLen / 2, &output1); + EXPECT_EQ(kChannels, output1.num_channels_); + EXPECT_EQ(kNewLen / 2, output1.samples_per_channel_); + + AudioFrame output2; + sync_buffer.GetNextAudioInterleaved(kNewLen / 2, &output2); + EXPECT_EQ(kChannels, output2.num_channels_); + EXPECT_EQ(kNewLen / 2, output2.samples_per_channel_); + + // Verify the data. + const int16_t* output_ptr = output1.data(); + for (size_t i = 0; i < kNewLen / 2; ++i) { + for (size_t channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(new_data[channel][i], *output_ptr); + ++output_ptr; + } + } + output_ptr = output2.data(); + for (size_t i = kNewLen / 2; i < kNewLen; ++i) { + for (size_t channel = 0; channel < kChannels; ++channel) { + EXPECT_EQ(new_data[channel][i], *output_ptr); + ++output_ptr; + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/delay_tool/parse_delay_file.m b/third_party/libwebrtc/modules/audio_coding/neteq/test/delay_tool/parse_delay_file.m new file mode 100644 index 0000000000..031d8a39ee --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/delay_tool/parse_delay_file.m @@ -0,0 +1,201 @@ +% +% Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +function outStruct = parse_delay_file(file) + +fid = fopen(file, 'rb'); +if fid == -1 + error('Cannot open file %s', file); +end + +textline = fgetl(fid); +if ~strncmp(textline, '#!NetEQ_Delay_Logging', 21) + error('Wrong file format'); +end + +ver = sscanf(textline, '#!NetEQ_Delay_Logging%d.%d'); +if ~all(ver == [2; 0]) + error('Wrong version of delay logging function') +end + + +start_pos = ftell(fid); +fseek(fid, -12, 'eof'); +textline = fgetl(fid); +if ~strncmp(textline, 'End of file', 21) + error('File ending is not correct. Seems like the simulation ended abnormally.'); +end + +fseek(fid,-12-4, 'eof'); +Npackets = fread(fid, 1, 'int32'); +fseek(fid, start_pos, 'bof'); + +rtpts = zeros(Npackets, 1); +seqno = zeros(Npackets, 1); +pt = zeros(Npackets, 1); +plen = zeros(Npackets, 1); +recin_t = nan*ones(Npackets, 1); +decode_t = nan*ones(Npackets, 1); +playout_delay = zeros(Npackets, 1); +optbuf = zeros(Npackets, 1); + +fs_ix = 1; +clock = 0; +ts_ix = 1; +ended = 0; +late_packets = 0; +fs_now = 8000; +last_decode_k = 0; +tot_expand = 0; +tot_accelerate = 0; +tot_preemptive = 0; + +while not(ended) + signal = fread(fid, 1, '*int32'); + + switch signal + case 3 % NETEQ_DELAY_LOGGING_SIGNAL_CLOCK + clock = fread(fid, 1, '*float32'); + + % keep on reading batches of M until the signal is no longer "3" + % read int32 + float32 in one go + % this is to save execution time + temp = [3; 0]; + M = 120; + while all(temp(1,:) == 3) + fp = ftell(fid); + temp = fread(fid, [2 M], '*int32'); + end + + % back up to last clock event + fseek(fid, fp - ftell(fid) + ... + (find(temp(1,:) ~= 3, 1 ) - 2) * 2 * 4 + 4, 'cof'); + % read the last clock value + clock = fread(fid, 1, '*float32'); + + case 1 % NETEQ_DELAY_LOGGING_SIGNAL_RECIN + temp_ts = fread(fid, 1, 'uint32'); + + if late_packets > 0 + temp_ix = ts_ix - 1; + while (temp_ix >= 1) && (rtpts(temp_ix) ~= temp_ts) + % TODO(hlundin): use matlab vector search instead? + temp_ix = temp_ix - 1; + end + + if temp_ix >= 1 + % the ts was found in the vector + late_packets = late_packets - 1; + else + temp_ix = ts_ix; + ts_ix = ts_ix + 1; + end + else + temp_ix = ts_ix; + ts_ix = ts_ix + 1; + end + + rtpts(temp_ix) = temp_ts; + seqno(temp_ix) = fread(fid, 1, 'uint16'); + pt(temp_ix) = fread(fid, 1, 'int32'); + plen(temp_ix) = fread(fid, 1, 'int16'); + recin_t(temp_ix) = clock; + + case 2 % NETEQ_DELAY_LOGGING_SIGNAL_FLUSH + % do nothing + + case 4 % NETEQ_DELAY_LOGGING_SIGNAL_EOF + ended = 1; + + case 5 % NETEQ_DELAY_LOGGING_SIGNAL_DECODE + last_decode_ts = fread(fid, 1, 'uint32'); + temp_delay = fread(fid, 1, 'uint16'); + + k = find(rtpts(1:(ts_ix - 1))==last_decode_ts,1,'last'); + if ~isempty(k) + decode_t(k) = clock; + playout_delay(k) = temp_delay + ... + 5 * fs_now / 8000; % add overlap length + last_decode_k = k; + end + + case 6 % NETEQ_DELAY_LOGGING_SIGNAL_CHANGE_FS + fsvec(fs_ix) = fread(fid, 1, 'uint16'); + fschange_ts(fs_ix) = last_decode_ts; + fs_now = fsvec(fs_ix); + fs_ix = fs_ix + 1; + + case 7 % NETEQ_DELAY_LOGGING_SIGNAL_MERGE_INFO + playout_delay(last_decode_k) = playout_delay(last_decode_k) ... + + fread(fid, 1, 'int32'); + + case 8 % NETEQ_DELAY_LOGGING_SIGNAL_EXPAND_INFO + temp = fread(fid, 1, 'int32'); + if last_decode_k ~= 0 + tot_expand = tot_expand + temp / (fs_now / 1000); + end + + case 9 % NETEQ_DELAY_LOGGING_SIGNAL_ACCELERATE_INFO + temp = fread(fid, 1, 'int32'); + if last_decode_k ~= 0 + tot_accelerate = tot_accelerate + temp / (fs_now / 1000); + end + + case 10 % NETEQ_DELAY_LOGGING_SIGNAL_PREEMPTIVE_INFO + temp = fread(fid, 1, 'int32'); + if last_decode_k ~= 0 + tot_preemptive = tot_preemptive + temp / (fs_now / 1000); + end + + case 11 % NETEQ_DELAY_LOGGING_SIGNAL_OPTBUF + optbuf(last_decode_k) = fread(fid, 1, 'int32'); + + case 12 % NETEQ_DELAY_LOGGING_SIGNAL_DECODE_ONE_DESC + last_decode_ts = fread(fid, 1, 'uint32'); + k = ts_ix - 1; + + while (k >= 1) && (rtpts(k) ~= last_decode_ts) + % TODO(hlundin): use matlab vector search instead? + k = k - 1; + end + + if k < 1 + % packet not received yet + k = ts_ix; + rtpts(ts_ix) = last_decode_ts; + late_packets = late_packets + 1; + end + + decode_t(k) = clock; + playout_delay(k) = fread(fid, 1, 'uint16') + ... + 5 * fs_now / 8000; % add overlap length + last_decode_k = k; + + end + +end + + +fclose(fid); + +outStruct = struct(... + 'ts', rtpts, ... + 'sn', seqno, ... + 'pt', pt,... + 'plen', plen,... + 'arrival', recin_t,... + 'decode', decode_t,... + 'fs', fsvec(:),... + 'fschange_ts', fschange_ts(:),... + 'playout_delay', playout_delay,... + 'tot_expand', tot_expand,... + 'tot_accelerate', tot_accelerate,... + 'tot_preemptive', tot_preemptive,... + 'optbuf', optbuf); diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/delay_tool/plot_neteq_delay.m b/third_party/libwebrtc/modules/audio_coding/neteq/test/delay_tool/plot_neteq_delay.m new file mode 100644 index 0000000000..86d533fbeb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/delay_tool/plot_neteq_delay.m @@ -0,0 +1,197 @@ +% +% Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +% +% Use of this source code is governed by a BSD-style license +% that can be found in the LICENSE file in the root of the source +% tree. An additional intellectual property rights grant can be found +% in the file PATENTS. All contributing project authors may +% be found in the AUTHORS file in the root of the source tree. +% + +function [delay_struct, delayvalues] = plot_neteq_delay(delayfile, varargin) + +% InfoStruct = plot_neteq_delay(delayfile) +% InfoStruct = plot_neteq_delay(delayfile, 'skipdelay', skip_seconds) +% +% Henrik Lundin, 2006-11-17 +% Henrik Lundin, 2011-05-17 +% + +try + s = parse_delay_file(delayfile); +catch + error(lasterr); +end + +delayskip=0; +noplot=0; +arg_ptr=1; +delaypoints=[]; + +s.sn=unwrap_seqno(s.sn); + +while arg_ptr+1 <= nargin + switch lower(varargin{arg_ptr}) + case {'skipdelay', 'delayskip'} + % skip a number of seconds in the beginning when calculating delays + delayskip = varargin{arg_ptr+1}; + arg_ptr = arg_ptr + 2; + case 'noplot' + noplot=1; + arg_ptr = arg_ptr + 1; + case {'get_delay', 'getdelay'} + % return a vector of delay values for the points in the given vector + delaypoints = varargin{arg_ptr+1}; + arg_ptr = arg_ptr + 2; + otherwise + warning('Unknown switch %s\n', varargin{arg_ptr}); + arg_ptr = arg_ptr + 1; + end +end + +% find lost frames that were covered by one-descriptor decoding +one_desc_ix=find(isnan(s.arrival)); +for k=1:length(one_desc_ix) + ix=find(s.ts==max(s.ts(s.ts(one_desc_ix(k))>s.ts))); + s.sn(one_desc_ix(k))=s.sn(ix)+1; + s.pt(one_desc_ix(k))=s.pt(ix); + s.arrival(one_desc_ix(k))=s.arrival(ix)+s.decode(one_desc_ix(k))-s.decode(ix); +end + +% remove duplicate received frames that were never decoded (RED codec) +if length(unique(s.ts(isfinite(s.ts)))) < length(s.ts(isfinite(s.ts))) + ix=find(isfinite(s.decode)); + s.sn=s.sn(ix); + s.ts=s.ts(ix); + s.arrival=s.arrival(ix); + s.playout_delay=s.playout_delay(ix); + s.pt=s.pt(ix); + s.optbuf=s.optbuf(ix); + plen=plen(ix); + s.decode=s.decode(ix); +end + +% find non-unique sequence numbers +[~,un_ix]=unique(s.sn); +nonun_ix=setdiff(1:length(s.sn),un_ix); +if ~isempty(nonun_ix) + warning('RTP sequence numbers are in error'); +end + +% sort vectors +[s.sn,sort_ix]=sort(s.sn); +s.ts=s.ts(sort_ix); +s.arrival=s.arrival(sort_ix); +s.decode=s.decode(sort_ix); +s.playout_delay=s.playout_delay(sort_ix); +s.pt=s.pt(sort_ix); + +send_t=s.ts-s.ts(1); +if length(s.fs)<1 + warning('No info about sample rate found in file. Using default 8000.'); + s.fs(1)=8000; + s.fschange_ts(1)=min(s.ts); +elseif s.fschange_ts(1)>min(s.ts) + s.fschange_ts(1)=min(s.ts); +end + +end_ix=length(send_t); +for k=length(s.fs):-1:1 + start_ix=find(s.ts==s.fschange_ts(k)); + send_t(start_ix:end_ix)=send_t(start_ix:end_ix)/s.fs(k)*1000; + s.playout_delay(start_ix:end_ix)=s.playout_delay(start_ix:end_ix)/s.fs(k)*1000; + s.optbuf(start_ix:end_ix)=s.optbuf(start_ix:end_ix)/s.fs(k)*1000; + end_ix=start_ix-1; +end + +tot_time=max(send_t)-min(send_t); + +seq_ix=s.sn-min(s.sn)+1; +send_t=send_t+max(min(s.arrival-send_t),0); + +plot_send_t=nan*ones(max(seq_ix),1); +plot_send_t(seq_ix)=send_t; +plot_nw_delay=nan*ones(max(seq_ix),1); +plot_nw_delay(seq_ix)=s.arrival-send_t; + +cng_ix=find(s.pt~=13); % find those packets that are not CNG/SID + +if noplot==0 + h=plot(plot_send_t/1000,plot_nw_delay); + set(h,'color',0.75*[1 1 1]); + hold on + if any(s.optbuf~=0) + peak_ix=find(s.optbuf(cng_ix)<0); % peak mode is labeled with negative values + no_peak_ix=find(s.optbuf(cng_ix)>0); %setdiff(1:length(cng_ix),peak_ix); + h1=plot(send_t(cng_ix(peak_ix))/1000,... + s.arrival(cng_ix(peak_ix))+abs(s.optbuf(cng_ix(peak_ix)))-send_t(cng_ix(peak_ix)),... + 'r.'); + h2=plot(send_t(cng_ix(no_peak_ix))/1000,... + s.arrival(cng_ix(no_peak_ix))+abs(s.optbuf(cng_ix(no_peak_ix)))-send_t(cng_ix(no_peak_ix)),... + 'g.'); + set([h1, h2],'markersize',1) + end + %h=plot(send_t(seq_ix)/1000,s.decode+s.playout_delay-send_t(seq_ix)); + h=plot(send_t(cng_ix)/1000,s.decode(cng_ix)+s.playout_delay(cng_ix)-send_t(cng_ix)); + set(h,'linew',1.5); + hold off + ax1=axis; + axis tight + ax2=axis; + axis([ax2(1:3) ax1(4)]) +end + + +% calculate delays and other parameters + +delayskip_ix = find(send_t-send_t(1)>=delayskip*1000, 1 ); + +use_ix = intersect(cng_ix,... % use those that are not CNG/SID frames... + intersect(find(isfinite(s.decode)),... % ... that did arrive ... + (delayskip_ix:length(s.decode))')); % ... and are sent after delayskip seconds + +mean_delay = mean(s.decode(use_ix)+s.playout_delay(use_ix)-send_t(use_ix)); +neteq_delay = mean(s.decode(use_ix)+s.playout_delay(use_ix)-s.arrival(use_ix)); + +Npack=max(s.sn(delayskip_ix:end))-min(s.sn(delayskip_ix:end))+1; +nw_lossrate=(Npack-length(s.sn(delayskip_ix:end)))/Npack; +neteq_lossrate=(length(s.sn(delayskip_ix:end))-length(use_ix))/Npack; + +delay_struct=struct('mean_delay',mean_delay,'neteq_delay',neteq_delay,... + 'nw_lossrate',nw_lossrate,'neteq_lossrate',neteq_lossrate,... + 'tot_expand',round(s.tot_expand),'tot_accelerate',round(s.tot_accelerate),... + 'tot_preemptive',round(s.tot_preemptive),'tot_time',tot_time,... + 'filename',delayfile,'units','ms','fs',unique(s.fs)); + +if not(isempty(delaypoints)) + delayvalues=interp1(send_t(cng_ix),... + s.decode(cng_ix)+s.playout_delay(cng_ix)-send_t(cng_ix),... + delaypoints,'nearest',NaN); +else + delayvalues=[]; +end + + + +% SUBFUNCTIONS % + +function y=unwrap_seqno(x) + +jumps=find(abs((diff(x)-1))>65000); + +while ~isempty(jumps) + n=jumps(1); + if x(n+1)-x(n) < 0 + % negative jump + x(n+1:end)=x(n+1:end)+65536; + else + % positive jump + x(n+1:end)=x(n+1:end)-65536; + end + + jumps=find(abs((diff(x(n+1:end))-1))>65000); +end + +y=x; + +return; diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_decoding_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_decoding_test.cc new file mode 100644 index 0000000000..e6c1809fb6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_decoding_test.cc @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/test/neteq_decoding_test.h" + +#include "absl/strings/string_view.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/rtp_headers.h" +#include "modules/audio_coding/neteq/default_neteq_factory.h" +#include "modules/audio_coding/neteq/test/result_sink.h" +#include "rtc_base/strings/string_builder.h" +#include "test/testsupport/file_utils.h" + +#ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.pb.h" +#else +#include "modules/audio_coding/neteq/neteq_unittest.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +namespace { + +void LoadDecoders(webrtc::NetEq* neteq) { + ASSERT_EQ(true, + neteq->RegisterPayloadType(0, SdpAudioFormat("pcmu", 8000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(8, SdpAudioFormat("pcma", 8000, 1))); +#ifdef WEBRTC_CODEC_ILBC + ASSERT_EQ(true, + neteq->RegisterPayloadType(102, SdpAudioFormat("ilbc", 8000, 1))); +#endif +#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX) + ASSERT_EQ(true, + neteq->RegisterPayloadType(103, SdpAudioFormat("isac", 16000, 1))); +#endif +#ifdef WEBRTC_CODEC_ISAC + ASSERT_EQ(true, + neteq->RegisterPayloadType(104, SdpAudioFormat("isac", 32000, 1))); +#endif +#ifdef WEBRTC_CODEC_OPUS + ASSERT_EQ(true, + neteq->RegisterPayloadType( + 111, SdpAudioFormat("opus", 48000, 2, {{"stereo", "0"}}))); +#endif + ASSERT_EQ(true, + neteq->RegisterPayloadType(93, SdpAudioFormat("L16", 8000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(94, SdpAudioFormat("L16", 16000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(95, SdpAudioFormat("L16", 32000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(13, SdpAudioFormat("cn", 8000, 1))); + ASSERT_EQ(true, + neteq->RegisterPayloadType(98, SdpAudioFormat("cn", 16000, 1))); +} + +} // namespace + +const int NetEqDecodingTest::kTimeStepMs; +const size_t NetEqDecodingTest::kBlockSize8kHz; +const size_t NetEqDecodingTest::kBlockSize16kHz; +const size_t NetEqDecodingTest::kBlockSize32kHz; +const int NetEqDecodingTest::kInitSampleRateHz; + +NetEqDecodingTest::NetEqDecodingTest() + : clock_(0), + config_(), + output_sample_rate_(kInitSampleRateHz), + algorithmic_delay_ms_(0) { + config_.sample_rate_hz = kInitSampleRateHz; +} + +void NetEqDecodingTest::SetUp() { + auto decoder_factory = CreateBuiltinAudioDecoderFactory(); + neteq_ = DefaultNetEqFactory().CreateNetEq(config_, decoder_factory, &clock_); + NetEqNetworkStatistics stat; + ASSERT_EQ(0, neteq_->NetworkStatistics(&stat)); + algorithmic_delay_ms_ = stat.current_buffer_size_ms; + ASSERT_TRUE(neteq_); + LoadDecoders(neteq_.get()); +} + +void NetEqDecodingTest::TearDown() {} + +void NetEqDecodingTest::OpenInputFile(absl::string_view rtp_file) { + rtp_source_.reset(test::RtpFileSource::Create(rtp_file)); +} + +void NetEqDecodingTest::Process() { + // Check if time to receive. + while (packet_ && clock_.TimeInMilliseconds() >= packet_->time_ms()) { + if (packet_->payload_length_bytes() > 0) { +#ifndef WEBRTC_CODEC_ISAC + // Ignore payload type 104 (iSAC-swb) if ISAC is not supported. + if (packet_->header().payloadType != 104) +#endif + ASSERT_EQ( + 0, neteq_->InsertPacket( + packet_->header(), + rtc::ArrayView( + packet_->payload(), packet_->payload_length_bytes()))); + } + // Get next packet. + packet_ = rtp_source_->NextPacket(); + } + + // Get audio from NetEq. + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_FALSE(muted); + ASSERT_TRUE((out_frame_.samples_per_channel_ == kBlockSize8kHz) || + (out_frame_.samples_per_channel_ == kBlockSize16kHz) || + (out_frame_.samples_per_channel_ == kBlockSize32kHz) || + (out_frame_.samples_per_channel_ == kBlockSize48kHz)); + output_sample_rate_ = out_frame_.sample_rate_hz_; + EXPECT_EQ(output_sample_rate_, neteq_->last_output_sample_rate_hz()); + + // Increase time. + clock_.AdvanceTimeMilliseconds(kTimeStepMs); +} + +void NetEqDecodingTest::DecodeAndCompare( + absl::string_view rtp_file, + absl::string_view output_checksum, + absl::string_view network_stats_checksum, + bool gen_ref) { + OpenInputFile(rtp_file); + + std::string ref_out_file = + gen_ref ? webrtc::test::OutputPath() + "neteq_universal_ref.pcm" : ""; + ResultSink output(ref_out_file); + + std::string stat_out_file = + gen_ref ? webrtc::test::OutputPath() + "neteq_network_stats.dat" : ""; + ResultSink network_stats(stat_out_file); + + packet_ = rtp_source_->NextPacket(); + int i = 0; + uint64_t last_concealed_samples = 0; + uint64_t last_total_samples_received = 0; + while (packet_) { + rtc::StringBuilder ss; + ss << "Lap number " << i++ << " in DecodeAndCompare while loop"; + SCOPED_TRACE(ss.str()); // Print out the parameter values on failure. + ASSERT_NO_FATAL_FAILURE(Process()); + ASSERT_NO_FATAL_FAILURE( + output.AddResult(out_frame_.data(), out_frame_.samples_per_channel_)); + + // Query the network statistics API once per second + if (clock_.TimeInMilliseconds() % 1000 == 0) { + // Process NetworkStatistics. + NetEqNetworkStatistics current_network_stats; + ASSERT_EQ(0, neteq_->NetworkStatistics(¤t_network_stats)); + ASSERT_NO_FATAL_FAILURE(network_stats.AddResult(current_network_stats)); + + // Verify that liftime stats and network stats report similar loss + // concealment rates. + auto lifetime_stats = neteq_->GetLifetimeStatistics(); + const uint64_t delta_concealed_samples = + lifetime_stats.concealed_samples - last_concealed_samples; + last_concealed_samples = lifetime_stats.concealed_samples; + const uint64_t delta_total_samples_received = + lifetime_stats.total_samples_received - last_total_samples_received; + last_total_samples_received = lifetime_stats.total_samples_received; + // The tolerance is 1% but expressed in Q14. + EXPECT_NEAR( + (delta_concealed_samples << 14) / delta_total_samples_received, + current_network_stats.expand_rate, (2 << 14) / 100.0); + } + } + + SCOPED_TRACE("Check output audio."); + output.VerifyChecksum(output_checksum); + SCOPED_TRACE("Check network stats."); + network_stats.VerifyChecksum(network_stats_checksum); +} + +void NetEqDecodingTest::PopulateRtpInfo(int frame_index, + int timestamp, + RTPHeader* rtp_info) { + rtp_info->sequenceNumber = frame_index; + rtp_info->timestamp = timestamp; + rtp_info->ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info->payloadType = 94; // PCM16b WB codec. + rtp_info->markerBit = false; +} + +void NetEqDecodingTest::PopulateCng(int frame_index, + int timestamp, + RTPHeader* rtp_info, + uint8_t* payload, + size_t* payload_len) { + rtp_info->sequenceNumber = frame_index; + rtp_info->timestamp = timestamp; + rtp_info->ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info->payloadType = 98; // WB CNG. + rtp_info->markerBit = false; + payload[0] = 64; // Noise level -64 dBov, quite arbitrarily chosen. + *payload_len = 1; // Only noise level, no spectral parameters. +} + +void NetEqDecodingTest::WrapTest(uint16_t start_seq_no, + uint32_t start_timestamp, + const std::set& drop_seq_numbers, + bool expect_seq_no_wrap, + bool expect_timestamp_wrap) { + uint16_t seq_no = start_seq_no; + uint32_t timestamp = start_timestamp; + const int kBlocksPerFrame = 3; // Number of 10 ms blocks per frame. + const int kFrameSizeMs = kBlocksPerFrame * kTimeStepMs; + const int kSamples = kBlockSize16kHz * kBlocksPerFrame; + const size_t kPayloadBytes = kSamples * sizeof(int16_t); + double next_input_time_ms = 0.0; + + // Insert speech for 2 seconds. + const int kSpeechDurationMs = 2000; + uint16_t last_seq_no; + uint32_t last_timestamp; + bool timestamp_wrapped = false; + bool seq_no_wrapped = false; + for (double t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) { + // Each turn in this for loop is 10 ms. + while (next_input_time_ms <= t_ms) { + // Insert one 30 ms speech frame. + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + if (drop_seq_numbers.find(seq_no) == drop_seq_numbers.end()) { + // This sequence number was not in the set to drop. Insert it. + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + } + NetEqNetworkStatistics network_stats; + ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats)); + + EXPECT_LE(network_stats.preferred_buffer_size_ms, 80); + EXPECT_LE(network_stats.current_buffer_size_ms, + 80 + algorithmic_delay_ms_); + last_seq_no = seq_no; + last_timestamp = timestamp; + + ++seq_no; + timestamp += kSamples; + next_input_time_ms += static_cast(kFrameSizeMs); + + seq_no_wrapped |= seq_no < last_seq_no; + timestamp_wrapped |= timestamp < last_timestamp; + } + // Pull out data once. + AudioFrame output; + bool muted; + ASSERT_EQ(0, neteq_->GetAudio(&output, &muted)); + ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_); + ASSERT_EQ(1u, output.num_channels_); + + // Expect delay (in samples) to be less than 2 packets. + absl::optional playout_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + EXPECT_LE(timestamp - *playout_timestamp, + static_cast(kSamples * 2)); + } + // Make sure we have actually tested wrap-around. + ASSERT_EQ(expect_seq_no_wrap, seq_no_wrapped); + ASSERT_EQ(expect_timestamp_wrap, timestamp_wrapped); +} + +void NetEqDecodingTest::LongCngWithClockDrift(double drift_factor, + double network_freeze_ms, + bool pull_audio_during_freeze, + int delay_tolerance_ms, + int max_time_to_speech_ms) { + uint16_t seq_no = 0; + uint32_t timestamp = 0; + const int kFrameSizeMs = 30; + const size_t kSamples = kFrameSizeMs * 16; + const size_t kPayloadBytes = kSamples * 2; + double next_input_time_ms = 0.0; + double t_ms; + bool muted; + + // Insert speech for 5 seconds. + const int kSpeechDurationMs = 5000; + for (t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) { + // Each turn in this for loop is 10 ms. + while (next_input_time_ms <= t_ms) { + // Insert one 30 ms speech frame. + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + ++seq_no; + timestamp += kSamples; + next_input_time_ms += static_cast(kFrameSizeMs) * drift_factor; + } + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + + EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_); + absl::optional playout_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + int32_t delay_before = timestamp - *playout_timestamp; + + // Insert CNG for 1 minute (= 60000 ms). + const int kCngPeriodMs = 100; + const int kCngPeriodSamples = kCngPeriodMs * 16; // Period in 16 kHz samples. + const int kCngDurationMs = 60000; + for (; t_ms < kSpeechDurationMs + kCngDurationMs; t_ms += 10) { + // Each turn in this for loop is 10 ms. + while (next_input_time_ms <= t_ms) { + // Insert one CNG frame each 100 ms. + uint8_t payload[kPayloadBytes]; + size_t payload_len; + RTPHeader rtp_info; + PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView( + payload, payload_len))); + ++seq_no; + timestamp += kCngPeriodSamples; + next_input_time_ms += static_cast(kCngPeriodMs) * drift_factor; + } + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + } + + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + + if (network_freeze_ms > 0) { + // First keep pulling audio for `network_freeze_ms` without inserting + // any data, then insert CNG data corresponding to `network_freeze_ms` + // without pulling any output audio. + const double loop_end_time = t_ms + network_freeze_ms; + for (; t_ms < loop_end_time; t_ms += 10) { + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + } + bool pull_once = pull_audio_during_freeze; + // If `pull_once` is true, GetAudio will be called once half-way through + // the network recovery period. + double pull_time_ms = (t_ms + next_input_time_ms) / 2; + while (next_input_time_ms <= t_ms) { + if (pull_once && next_input_time_ms >= pull_time_ms) { + pull_once = false; + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_); + t_ms += 10; + } + // Insert one CNG frame each 100 ms. + uint8_t payload[kPayloadBytes]; + size_t payload_len; + RTPHeader rtp_info; + PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView( + payload, payload_len))); + ++seq_no; + timestamp += kCngPeriodSamples; + next_input_time_ms += kCngPeriodMs * drift_factor; + } + } + + // Insert speech again until output type is speech. + double speech_restart_time_ms = t_ms; + while (out_frame_.speech_type_ != AudioFrame::kNormalSpeech) { + // Each turn in this for loop is 10 ms. + while (next_input_time_ms <= t_ms) { + // Insert one 30 ms speech frame. + uint8_t payload[kPayloadBytes] = {0}; + RTPHeader rtp_info; + PopulateRtpInfo(seq_no, timestamp, &rtp_info); + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload)); + ++seq_no; + timestamp += kSamples; + next_input_time_ms += kFrameSizeMs * drift_factor; + } + // Pull out data once. + ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted)); + ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_); + // Increase clock. + t_ms += 10; + } + + // Check that the speech starts again within reasonable time. + double time_until_speech_returns_ms = t_ms - speech_restart_time_ms; + EXPECT_LT(time_until_speech_returns_ms, max_time_to_speech_ms); + playout_timestamp = neteq_->GetPlayoutTimestamp(); + ASSERT_TRUE(playout_timestamp); + int32_t delay_after = timestamp - *playout_timestamp; + // Compare delay before and after, and make sure it differs less than 20 ms. + EXPECT_LE(delay_after, delay_before + delay_tolerance_ms * 16); + EXPECT_GE(delay_after, delay_before - delay_tolerance_ms * 16); +} + +void NetEqDecodingTestTwoInstances::SetUp() { + NetEqDecodingTest::SetUp(); + config2_ = config_; +} + +void NetEqDecodingTestTwoInstances::CreateSecondInstance() { + auto decoder_factory = CreateBuiltinAudioDecoderFactory(); + neteq2_ = + DefaultNetEqFactory().CreateNetEq(config2_, decoder_factory, &clock_); + ASSERT_TRUE(neteq2_); + LoadDecoders(neteq2_.get()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_decoding_test.h b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_decoding_test.h new file mode 100644 index 0000000000..456c397fdd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_decoding_test.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TEST_NETEQ_DECODING_TEST_H_ +#define MODULES_AUDIO_CODING_NETEQ_TEST_NETEQ_DECODING_TEST_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/audio/audio_frame.h" +#include "api/neteq/neteq.h" +#include "api/rtp_headers.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" + +namespace webrtc { + +class NetEqDecodingTest : public ::testing::Test { + protected: + // NetEQ must be polled for data once every 10 ms. + // Thus, none of the constants below can be changed. + static constexpr int kTimeStepMs = 10; + static constexpr size_t kBlockSize8kHz = kTimeStepMs * 8; + static constexpr size_t kBlockSize16kHz = kTimeStepMs * 16; + static constexpr size_t kBlockSize32kHz = kTimeStepMs * 32; + static constexpr size_t kBlockSize48kHz = kTimeStepMs * 48; + static constexpr int kInitSampleRateHz = 8000; + + NetEqDecodingTest(); + virtual void SetUp(); + virtual void TearDown(); + void OpenInputFile(absl::string_view rtp_file); + void Process(); + + void DecodeAndCompare(absl::string_view rtp_file, + absl::string_view output_checksum, + absl::string_view network_stats_checksum, + bool gen_ref); + + static void PopulateRtpInfo(int frame_index, + int timestamp, + RTPHeader* rtp_info); + static void PopulateCng(int frame_index, + int timestamp, + RTPHeader* rtp_info, + uint8_t* payload, + size_t* payload_len); + + void WrapTest(uint16_t start_seq_no, + uint32_t start_timestamp, + const std::set& drop_seq_numbers, + bool expect_seq_no_wrap, + bool expect_timestamp_wrap); + + void LongCngWithClockDrift(double drift_factor, + double network_freeze_ms, + bool pull_audio_during_freeze, + int delay_tolerance_ms, + int max_time_to_speech_ms); + + SimulatedClock clock_; + std::unique_ptr neteq_; + NetEq::Config config_; + std::unique_ptr rtp_source_; + std::unique_ptr packet_; + AudioFrame out_frame_; + int output_sample_rate_; + int algorithmic_delay_ms_; +}; + +class NetEqDecodingTestTwoInstances : public NetEqDecodingTest { + public: + NetEqDecodingTestTwoInstances() : NetEqDecodingTest() {} + + void SetUp() override; + + void CreateSecondInstance(); + + protected: + std::unique_ptr neteq2_; + NetEq::Config config2_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TEST_NETEQ_DECODING_TEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_ilbc_quality_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_ilbc_quality_test.cc new file mode 100644 index 0000000000..1004141f16 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_ilbc_quality_test.cc @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "absl/flags/flag.h" +#include "modules/audio_coding/codecs/ilbc/audio_encoder_ilbc.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/testsupport/file_utils.h" + +ABSL_FLAG(int, frame_size_ms, 20, "Codec frame size (milliseconds)."); + +using ::testing::InitGoogleTest; + +namespace webrtc { +namespace test { +namespace { +static const int kInputSampleRateKhz = 8; +static const int kOutputSampleRateKhz = 8; +} // namespace + +class NetEqIlbcQualityTest : public NetEqQualityTest { + protected: + NetEqIlbcQualityTest() + : NetEqQualityTest(absl::GetFlag(FLAGS_frame_size_ms), + kInputSampleRateKhz, + kOutputSampleRateKhz, + SdpAudioFormat("ilbc", 8000, 1)) { + // Flag validation + RTC_CHECK(absl::GetFlag(FLAGS_frame_size_ms) == 20 || + absl::GetFlag(FLAGS_frame_size_ms) == 30 || + absl::GetFlag(FLAGS_frame_size_ms) == 40 || + absl::GetFlag(FLAGS_frame_size_ms) == 60) + << "Invalid frame size, should be 20, 30, 40, or 60 ms."; + } + + void SetUp() override { + ASSERT_EQ(1u, channels_) << "iLBC supports only mono audio."; + AudioEncoderIlbcConfig config; + config.frame_size_ms = absl::GetFlag(FLAGS_frame_size_ms); + encoder_.reset(new AudioEncoderIlbcImpl(config, 102)); + NetEqQualityTest::SetUp(); + } + + int EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, + size_t max_bytes) override { + const size_t kFrameSizeSamples = 80; // Samples per 10 ms. + size_t encoded_samples = 0; + uint32_t dummy_timestamp = 0; + AudioEncoder::EncodedInfo info; + do { + info = encoder_->Encode(dummy_timestamp, + rtc::ArrayView( + in_data + encoded_samples, kFrameSizeSamples), + payload); + encoded_samples += kFrameSizeSamples; + } while (info.encoded_bytes == 0); + return rtc::checked_cast(info.encoded_bytes); + } + + private: + std::unique_ptr encoder_; +}; + +TEST_F(NetEqIlbcQualityTest, Test) { + Simulate(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc new file mode 100644 index 0000000000..5a2df24ef6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "absl/flags/flag.h" +#include "modules/audio_coding/codecs/opus/opus_inst.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" + +ABSL_FLAG(int, bit_rate_kbps, 32, "Target bit rate (kbps)."); + +ABSL_FLAG(int, + complexity, + 10, + "Complexity: 0 ~ 10 -- defined as in Opus" + "specification."); + +ABSL_FLAG(int, maxplaybackrate, 48000, "Maximum playback rate (Hz)."); + +ABSL_FLAG(int, application, 0, "Application mode: 0 -- VOIP, 1 -- Audio."); + +ABSL_FLAG(int, reported_loss_rate, 10, "Reported percentile of packet loss."); + +ABSL_FLAG(bool, fec, false, "Enable FEC for encoding (-nofec to disable)."); + +ABSL_FLAG(bool, dtx, false, "Enable DTX for encoding (-nodtx to disable)."); + +ABSL_FLAG(int, sub_packets, 1, "Number of sub packets to repacketize."); + +using ::testing::InitGoogleTest; + +namespace webrtc { +namespace test { +namespace { + +static const int kOpusBlockDurationMs = 20; +static const int kOpusSamplingKhz = 48; +} // namespace + +class NetEqOpusQualityTest : public NetEqQualityTest { + protected: + NetEqOpusQualityTest(); + void SetUp() override; + void TearDown() override; + int EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, + size_t max_bytes) override; + + private: + WebRtcOpusEncInst* opus_encoder_; + OpusRepacketizer* repacketizer_; + size_t sub_block_size_samples_; + int bit_rate_kbps_; + bool fec_; + bool dtx_; + int complexity_; + int maxplaybackrate_; + int target_loss_rate_; + int sub_packets_; + int application_; +}; + +NetEqOpusQualityTest::NetEqOpusQualityTest() + : NetEqQualityTest(kOpusBlockDurationMs * absl::GetFlag(FLAGS_sub_packets), + kOpusSamplingKhz, + kOpusSamplingKhz, + SdpAudioFormat("opus", 48000, 2)), + opus_encoder_(NULL), + repacketizer_(NULL), + sub_block_size_samples_( + static_cast(kOpusBlockDurationMs * kOpusSamplingKhz)), + bit_rate_kbps_(absl::GetFlag(FLAGS_bit_rate_kbps)), + fec_(absl::GetFlag(FLAGS_fec)), + dtx_(absl::GetFlag(FLAGS_dtx)), + complexity_(absl::GetFlag(FLAGS_complexity)), + maxplaybackrate_(absl::GetFlag(FLAGS_maxplaybackrate)), + target_loss_rate_(absl::GetFlag(FLAGS_reported_loss_rate)), + sub_packets_(absl::GetFlag(FLAGS_sub_packets)) { + // Flag validation + RTC_CHECK(absl::GetFlag(FLAGS_bit_rate_kbps) >= 6 && + absl::GetFlag(FLAGS_bit_rate_kbps) <= 510) + << "Invalid bit rate, should be between 6 and 510 kbps."; + + RTC_CHECK(absl::GetFlag(FLAGS_complexity) >= -1 && + absl::GetFlag(FLAGS_complexity) <= 10) + << "Invalid complexity setting, should be between 0 and 10."; + + RTC_CHECK(absl::GetFlag(FLAGS_application) == 0 || + absl::GetFlag(FLAGS_application) == 1) + << "Invalid application mode, should be 0 or 1."; + + RTC_CHECK(absl::GetFlag(FLAGS_reported_loss_rate) >= 0 && + absl::GetFlag(FLAGS_reported_loss_rate) <= 100) + << "Invalid packet loss percentile, should be between 0 and 100."; + + RTC_CHECK(absl::GetFlag(FLAGS_sub_packets) >= 1 && + absl::GetFlag(FLAGS_sub_packets) <= 3) + << "Invalid number of sub packets, should be between 1 and 3."; + + // Redefine decoder type if input is stereo. + if (channels_ > 1) { + audio_format_ = SdpAudioFormat("opus", 48000, 2, + SdpAudioFormat::Parameters{{"stereo", "1"}}); + } + application_ = absl::GetFlag(FLAGS_application); +} + +void NetEqOpusQualityTest::SetUp() { + // Create encoder memory. + WebRtcOpus_EncoderCreate(&opus_encoder_, channels_, application_, 48000); + ASSERT_TRUE(opus_encoder_); + + // Create repacketizer. + repacketizer_ = opus_repacketizer_create(); + ASSERT_TRUE(repacketizer_); + + // Set bitrate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_encoder_, bit_rate_kbps_ * 1000)); + if (fec_) { + EXPECT_EQ(0, WebRtcOpus_EnableFec(opus_encoder_)); + } + if (dtx_) { + EXPECT_EQ(0, WebRtcOpus_EnableDtx(opus_encoder_)); + } + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_encoder_, complexity_)); + EXPECT_EQ(0, WebRtcOpus_SetMaxPlaybackRate(opus_encoder_, maxplaybackrate_)); + EXPECT_EQ(0, WebRtcOpus_SetPacketLossRate(opus_encoder_, target_loss_rate_)); + NetEqQualityTest::SetUp(); +} + +void NetEqOpusQualityTest::TearDown() { + // Free memory. + EXPECT_EQ(0, WebRtcOpus_EncoderFree(opus_encoder_)); + opus_repacketizer_destroy(repacketizer_); + NetEqQualityTest::TearDown(); +} + +int NetEqOpusQualityTest::EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, + size_t max_bytes) { + EXPECT_EQ(block_size_samples, sub_block_size_samples_ * sub_packets_); + int16_t* pointer = in_data; + int value; + opus_repacketizer_init(repacketizer_); + for (int idx = 0; idx < sub_packets_; idx++) { + payload->AppendData(max_bytes, [&](rtc::ArrayView payload) { + value = WebRtcOpus_Encode(opus_encoder_, pointer, sub_block_size_samples_, + max_bytes, payload.data()); + + Log() << "Encoded a frame with Opus mode " + << (value == 0 ? 0 : payload[0] >> 3) << std::endl; + + return (value >= 0) ? static_cast(value) : 0; + }); + + if (OPUS_OK != + opus_repacketizer_cat(repacketizer_, payload->data(), value)) { + opus_repacketizer_init(repacketizer_); + // If the repacketization fails, we discard this frame. + return 0; + } + pointer += sub_block_size_samples_ * channels_; + } + value = opus_repacketizer_out(repacketizer_, payload->data(), + static_cast(max_bytes)); + EXPECT_GE(value, 0); + return value; +} + +TEST_F(NetEqOpusQualityTest, Test) { + Simulate(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_pcm16b_quality_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_pcm16b_quality_test.cc new file mode 100644 index 0000000000..c3e160cb66 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_pcm16b_quality_test.cc @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "absl/flags/flag.h" +#include "modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/testsupport/file_utils.h" + +ABSL_FLAG(int, frame_size_ms, 20, "Codec frame size (milliseconds)."); + +using ::testing::InitGoogleTest; + +namespace webrtc { +namespace test { +namespace { +static const int kInputSampleRateKhz = 48; +static const int kOutputSampleRateKhz = 48; +} // namespace + +class NetEqPcm16bQualityTest : public NetEqQualityTest { + protected: + NetEqPcm16bQualityTest() + : NetEqQualityTest(absl::GetFlag(FLAGS_frame_size_ms), + kInputSampleRateKhz, + kOutputSampleRateKhz, + SdpAudioFormat("l16", 48000, 1)) { + // Flag validation + RTC_CHECK(absl::GetFlag(FLAGS_frame_size_ms) >= 10 && + absl::GetFlag(FLAGS_frame_size_ms) <= 60 && + (absl::GetFlag(FLAGS_frame_size_ms) % 10) == 0) + << "Invalid frame size, should be 10, 20, ..., 60 ms."; + } + + void SetUp() override { + AudioEncoderPcm16B::Config config; + config.frame_size_ms = absl::GetFlag(FLAGS_frame_size_ms); + config.sample_rate_hz = 48000; + config.num_channels = channels_; + encoder_.reset(new AudioEncoderPcm16B(config)); + NetEqQualityTest::SetUp(); + } + + int EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, + size_t max_bytes) override { + const size_t kFrameSizeSamples = 480; // Samples per 10 ms. + size_t encoded_samples = 0; + uint32_t dummy_timestamp = 0; + AudioEncoder::EncodedInfo info; + do { + info = encoder_->Encode(dummy_timestamp, + rtc::ArrayView( + in_data + encoded_samples, kFrameSizeSamples), + payload); + encoded_samples += kFrameSizeSamples; + } while (info.encoded_bytes == 0); + return rtc::checked_cast(info.encoded_bytes); + } + + private: + std::unique_ptr encoder_; +}; + +TEST_F(NetEqPcm16bQualityTest, Test) { + Simulate(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_pcmu_quality_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_pcmu_quality_test.cc new file mode 100644 index 0000000000..d22170c623 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_pcmu_quality_test.cc @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "absl/flags/flag.h" +#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/testsupport/file_utils.h" + +ABSL_FLAG(int, frame_size_ms, 20, "Codec frame size (milliseconds)."); + +using ::testing::InitGoogleTest; + +namespace webrtc { +namespace test { +namespace { +static const int kInputSampleRateKhz = 8; +static const int kOutputSampleRateKhz = 8; +} // namespace + +class NetEqPcmuQualityTest : public NetEqQualityTest { + protected: + NetEqPcmuQualityTest() + : NetEqQualityTest(absl::GetFlag(FLAGS_frame_size_ms), + kInputSampleRateKhz, + kOutputSampleRateKhz, + SdpAudioFormat("pcmu", 8000, 1)) { + // Flag validation + RTC_CHECK(absl::GetFlag(FLAGS_frame_size_ms) >= 10 && + absl::GetFlag(FLAGS_frame_size_ms) <= 60 && + (absl::GetFlag(FLAGS_frame_size_ms) % 10) == 0) + << "Invalid frame size, should be 10, 20, ..., 60 ms."; + } + + void SetUp() override { + ASSERT_EQ(1u, channels_) << "PCMu supports only mono audio."; + AudioEncoderPcmU::Config config; + config.frame_size_ms = absl::GetFlag(FLAGS_frame_size_ms); + encoder_.reset(new AudioEncoderPcmU(config)); + NetEqQualityTest::SetUp(); + } + + int EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, + size_t max_bytes) override { + const size_t kFrameSizeSamples = 80; // Samples per 10 ms. + size_t encoded_samples = 0; + uint32_t dummy_timestamp = 0; + AudioEncoder::EncodedInfo info; + do { + info = encoder_->Encode(dummy_timestamp, + rtc::ArrayView( + in_data + encoded_samples, kFrameSizeSamples), + payload); + encoded_samples += kFrameSizeSamples; + } while (info.encoded_bytes == 0); + return rtc::checked_cast(info.encoded_bytes); + } + + private: + std::unique_ptr encoder_; +}; + +TEST_F(NetEqPcmuQualityTest, Test) { + Simulate(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_performance_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_performance_unittest.cc new file mode 100644 index 0000000000..961f74ab66 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_performance_unittest.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/metrics/metric.h" +#include "modules/audio_coding/neteq/tools/neteq_performance_test.h" +#include "system_wrappers/include/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::webrtc::test::GetGlobalMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + +// Runs a test with 10% packet losses and 10% clock drift, to exercise +// both loss concealment and time-stretching code. +TEST(NetEqPerformanceTest, 10_Pl_10_Drift) { + const int kSimulationTimeMs = 10000000; + const int kQuickSimulationTimeMs = 100000; + const int kLossPeriod = 10; // Drop every 10th packet. + const double kDriftFactor = 0.1; + int64_t runtime = test::NetEqPerformanceTest::Run( + field_trial::IsEnabled("WebRTC-QuickPerfTest") ? kQuickSimulationTimeMs + : kSimulationTimeMs, + kLossPeriod, kDriftFactor); + ASSERT_GT(runtime, 0); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "neteq_performance", "10_pl_10_drift", runtime, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter); +} + +// Runs a test with neither packet losses nor clock drift, to put +// emphasis on the "good-weather" code path, which is presumably much +// more lightweight. +TEST(NetEqPerformanceTest, 0_Pl_0_Drift) { + const int kSimulationTimeMs = 10000000; + const int kQuickSimulationTimeMs = 100000; + const int kLossPeriod = 0; // No losses. + const double kDriftFactor = 0.0; // No clock drift. + int64_t runtime = test::NetEqPerformanceTest::Run( + field_trial::IsEnabled("WebRTC-QuickPerfTest") ? kQuickSimulationTimeMs + : kSimulationTimeMs, + kLossPeriod, kDriftFactor); + ASSERT_GT(runtime, 0); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "neteq_performance", "0_pl_0_drift", runtime, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc new file mode 100644 index 0000000000..a72b2009eb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_speed_test.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "modules/audio_coding/neteq/tools/neteq_performance_test.h" +#include "rtc_base/checks.h" + +// Define command line flags. +ABSL_FLAG(int, runtime_ms, 10000, "Simulated runtime in ms."); +ABSL_FLAG(int, lossrate, 10, "Packet lossrate; drop every N packets."); +ABSL_FLAG(float, drift, 0.1f, "Clockdrift factor."); + +int main(int argc, char* argv[]) { + std::vector args = absl::ParseCommandLine(argc, argv); + std::string program_name = args[0]; + std::string usage = + "Tool for measuring the speed of NetEq.\n" + "Usage: " + + program_name + + " [options]\n\n" + " --runtime_ms=N runtime in ms; default is 10000 ms\n" + " --lossrate=N drop every N packets; default is 10\n" + " --drift=F clockdrift factor between 0.0 and 1.0; " + "default is 0.1\n"; + if (args.size() != 1) { + printf("%s", usage.c_str()); + return 1; + } + RTC_CHECK_GT(absl::GetFlag(FLAGS_runtime_ms), 0); + RTC_CHECK_GE(absl::GetFlag(FLAGS_lossrate), 0); + RTC_CHECK(absl::GetFlag(FLAGS_drift) >= 0.0 && + absl::GetFlag(FLAGS_drift) < 1.0); + + int64_t result = webrtc::test::NetEqPerformanceTest::Run( + absl::GetFlag(FLAGS_runtime_ms), absl::GetFlag(FLAGS_lossrate), + absl::GetFlag(FLAGS_drift)); + if (result <= 0) { + std::cout << "There was an error" << std::endl; + return -1; + } + + std::cout << "Simulation done" << std::endl; + std::cout << "Runtime = " << result << " ms" << std::endl; + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/result_sink.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/result_sink.cc new file mode 100644 index 0000000000..f5d50dc859 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/result_sink.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/test/result_sink.h" + +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/ignore_wundef.h" +#include "rtc_base/message_digest.h" +#include "rtc_base/string_encode.h" +#include "test/gtest.h" + +#ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT +RTC_PUSH_IGNORING_WUNDEF() +#ifdef WEBRTC_ANDROID_PLATFORM_BUILD +#include "external/webrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.pb.h" +#else +#include "modules/audio_coding/neteq/neteq_unittest.pb.h" +#endif +RTC_POP_IGNORING_WUNDEF() +#endif + +namespace webrtc { + +#ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT +void Convert(const webrtc::NetEqNetworkStatistics& stats_raw, + webrtc::neteq_unittest::NetEqNetworkStatistics* stats) { + stats->set_current_buffer_size_ms(stats_raw.current_buffer_size_ms); + stats->set_preferred_buffer_size_ms(stats_raw.preferred_buffer_size_ms); + stats->set_jitter_peaks_found(stats_raw.jitter_peaks_found); + stats->set_expand_rate(stats_raw.expand_rate); + stats->set_speech_expand_rate(stats_raw.speech_expand_rate); + stats->set_preemptive_rate(stats_raw.preemptive_rate); + stats->set_accelerate_rate(stats_raw.accelerate_rate); + stats->set_secondary_decoded_rate(stats_raw.secondary_decoded_rate); + stats->set_secondary_discarded_rate(stats_raw.secondary_discarded_rate); + stats->set_mean_waiting_time_ms(stats_raw.mean_waiting_time_ms); + stats->set_median_waiting_time_ms(stats_raw.median_waiting_time_ms); + stats->set_min_waiting_time_ms(stats_raw.min_waiting_time_ms); + stats->set_max_waiting_time_ms(stats_raw.max_waiting_time_ms); +} + +void AddMessage(FILE* file, + rtc::MessageDigest* digest, + absl::string_view message) { + int32_t size = message.length(); + if (file) + ASSERT_EQ(1u, fwrite(&size, sizeof(size), 1, file)); + digest->Update(&size, sizeof(size)); + + if (file) + ASSERT_EQ(static_cast(size), + fwrite(message.data(), sizeof(char), size, file)); + digest->Update(message.data(), sizeof(char) * size); +} + +#endif // WEBRTC_NETEQ_UNITTEST_BITEXACT + +ResultSink::ResultSink(absl::string_view output_file) + : output_fp_(nullptr), + digest_(rtc::MessageDigestFactory::Create(rtc::DIGEST_SHA_1)) { + if (!output_file.empty()) { + output_fp_ = fopen(std::string(output_file).c_str(), "wb"); + EXPECT_TRUE(output_fp_ != NULL); + } +} + +ResultSink::~ResultSink() { + if (output_fp_) + fclose(output_fp_); +} + +void ResultSink::AddResult(const NetEqNetworkStatistics& stats_raw) { +#ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT + neteq_unittest::NetEqNetworkStatistics stats; + Convert(stats_raw, &stats); + + std::string stats_string; + ASSERT_TRUE(stats.SerializeToString(&stats_string)); + AddMessage(output_fp_, digest_.get(), stats_string); +#else + FAIL() << "Writing to reference file requires Proto Buffer."; +#endif // WEBRTC_NETEQ_UNITTEST_BITEXACT +} + +void ResultSink::VerifyChecksum(absl::string_view checksum) { + std::string buffer; + buffer.resize(digest_->Size()); + digest_->Finish(buffer.data(), buffer.size()); + const std::string result = rtc::hex_encode(buffer); + if (checksum.size() == result.size()) { + EXPECT_EQ(checksum, result); + } else { + // Check result is one the '|'-separated checksums. + EXPECT_NE(checksum.find(result), absl::string_view::npos) + << result << " should be one of these:\n" + << checksum; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/result_sink.h b/third_party/libwebrtc/modules/audio_coding/neteq/test/result_sink.h new file mode 100644 index 0000000000..c6923d7a7f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/result_sink.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TEST_RESULT_SINK_H_ +#define MODULES_AUDIO_CODING_NETEQ_TEST_RESULT_SINK_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/neteq/neteq.h" +#include "rtc_base/message_digest.h" + +namespace webrtc { + +class ResultSink { + public: + explicit ResultSink(absl::string_view output_file); + ~ResultSink(); + + template + void AddResult(const T* test_results, size_t length); + + void AddResult(const NetEqNetworkStatistics& stats); + + void VerifyChecksum(absl::string_view ref_check_sum); + + private: + FILE* output_fp_; + std::unique_ptr digest_; +}; + +template +void ResultSink::AddResult(const T* test_results, size_t length) { + if (output_fp_) { + ASSERT_EQ(length, fwrite(test_results, sizeof(T), length, output_fp_)); + } + digest_->Update(test_results, sizeof(T) * length); +} + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TEST_RESULT_SINK_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.cc b/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.cc new file mode 100644 index 0000000000..b89be0608d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.cc @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/time_stretch.h" + +#include // min, max +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/cross_correlation.h" +#include "modules/audio_coding/neteq/dsp_helper.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +TimeStretch::ReturnCodes TimeStretch::Process(const int16_t* input, + size_t input_len, + bool fast_mode, + AudioMultiVector* output, + size_t* length_change_samples) { + // Pre-calculate common multiplication with `fs_mult_`. + size_t fs_mult_120 = + static_cast(fs_mult_ * 120); // Corresponds to 15 ms. + + const int16_t* signal; + std::unique_ptr signal_array; + size_t signal_len; + if (num_channels_ == 1) { + signal = input; + signal_len = input_len; + } else { + // We want `signal` to be only the first channel of `input`, which is + // interleaved. Thus, we take the first sample, skip forward `num_channels` + // samples, and continue like that. + signal_len = input_len / num_channels_; + signal_array.reset(new int16_t[signal_len]); + signal = signal_array.get(); + size_t j = kRefChannel; + for (size_t i = 0; i < signal_len; ++i) { + signal_array[i] = input[j]; + j += num_channels_; + } + } + + // Find maximum absolute value of input signal. + max_input_value_ = WebRtcSpl_MaxAbsValueW16(signal, signal_len); + + // Downsample to 4 kHz sample rate and calculate auto-correlation. + DspHelper::DownsampleTo4kHz(signal, signal_len, kDownsampledLen, + sample_rate_hz_, true /* compensate delay*/, + downsampled_input_); + AutoCorrelation(); + + // Find the strongest correlation peak. + static const size_t kNumPeaks = 1; + size_t peak_index; + int16_t peak_value; + DspHelper::PeakDetection(auto_correlation_, kCorrelationLen, kNumPeaks, + fs_mult_, &peak_index, &peak_value); + // Assert that `peak_index` stays within boundaries. + RTC_DCHECK_LE(peak_index, (2 * kCorrelationLen - 1) * fs_mult_); + + // Compensate peak_index for displaced starting position. The displacement + // happens in AutoCorrelation(). Here, `kMinLag` is in the down-sampled 4 kHz + // domain, while the `peak_index` is in the original sample rate; hence, the + // multiplication by fs_mult_ * 2. + peak_index += kMinLag * fs_mult_ * 2; + // Assert that `peak_index` stays within boundaries. + RTC_DCHECK_GE(peak_index, static_cast(20 * fs_mult_)); + RTC_DCHECK_LE(peak_index, + 20 * fs_mult_ + (2 * kCorrelationLen - 1) * fs_mult_); + + // Calculate scaling to ensure that `peak_index` samples can be square-summed + // without overflowing. + int scaling = 31 - WebRtcSpl_NormW32(max_input_value_ * max_input_value_) - + WebRtcSpl_NormW32(static_cast(peak_index)); + scaling = std::max(0, scaling); + + // `vec1` starts at 15 ms minus one pitch period. + const int16_t* vec1 = &signal[fs_mult_120 - peak_index]; + // `vec2` start at 15 ms. + const int16_t* vec2 = &signal[fs_mult_120]; + // Calculate energies for `vec1` and `vec2`, assuming they both contain + // `peak_index` samples. + int32_t vec1_energy = + WebRtcSpl_DotProductWithScale(vec1, vec1, peak_index, scaling); + int32_t vec2_energy = + WebRtcSpl_DotProductWithScale(vec2, vec2, peak_index, scaling); + + // Calculate cross-correlation between `vec1` and `vec2`. + int32_t cross_corr = + WebRtcSpl_DotProductWithScale(vec1, vec2, peak_index, scaling); + + // Check if the signal seems to be active speech or not (simple VAD). + bool active_speech = + SpeechDetection(vec1_energy, vec2_energy, peak_index, scaling); + + int16_t best_correlation; + if (!active_speech) { + SetParametersForPassiveSpeech(signal_len, &best_correlation, &peak_index); + } else { + // Calculate correlation: + // cross_corr / sqrt(vec1_energy * vec2_energy). + + // Start with calculating scale values. + int energy1_scale = std::max(0, 16 - WebRtcSpl_NormW32(vec1_energy)); + int energy2_scale = std::max(0, 16 - WebRtcSpl_NormW32(vec2_energy)); + + // Make sure total scaling is even (to simplify scale factor after sqrt). + if ((energy1_scale + energy2_scale) & 1) { + // The sum is odd. + energy1_scale += 1; + } + + // Scale energies to int16_t. + int16_t vec1_energy_int16 = + static_cast(vec1_energy >> energy1_scale); + int16_t vec2_energy_int16 = + static_cast(vec2_energy >> energy2_scale); + + // Calculate square-root of energy product. + int16_t sqrt_energy_prod = + WebRtcSpl_SqrtFloor(vec1_energy_int16 * vec2_energy_int16); + + // Calculate cross_corr / sqrt(en1*en2) in Q14. + int temp_scale = 14 - (energy1_scale + energy2_scale) / 2; + cross_corr = WEBRTC_SPL_SHIFT_W32(cross_corr, temp_scale); + cross_corr = std::max(0, cross_corr); // Don't use if negative. + best_correlation = WebRtcSpl_DivW32W16(cross_corr, sqrt_energy_prod); + // Make sure `best_correlation` is no larger than 1 in Q14. + best_correlation = std::min(static_cast(16384), best_correlation); + } + + // Check accelerate criteria and stretch the signal. + ReturnCodes return_value = + CheckCriteriaAndStretch(input, input_len, peak_index, best_correlation, + active_speech, fast_mode, output); + switch (return_value) { + case kSuccess: + *length_change_samples = peak_index; + break; + case kSuccessLowEnergy: + *length_change_samples = peak_index; + break; + case kNoStretch: + case kError: + *length_change_samples = 0; + break; + } + return return_value; +} + +void TimeStretch::AutoCorrelation() { + // Calculate correlation from lag kMinLag to lag kMaxLag in 4 kHz domain. + int32_t auto_corr[kCorrelationLen]; + CrossCorrelationWithAutoShift( + &downsampled_input_[kMaxLag], &downsampled_input_[kMaxLag - kMinLag], + kCorrelationLen, kMaxLag - kMinLag, -1, auto_corr); + + // Normalize correlation to 14 bits and write to `auto_correlation_`. + int32_t max_corr = WebRtcSpl_MaxAbsValueW32(auto_corr, kCorrelationLen); + int scaling = std::max(0, 17 - WebRtcSpl_NormW32(max_corr)); + WebRtcSpl_VectorBitShiftW32ToW16(auto_correlation_, kCorrelationLen, + auto_corr, scaling); +} + +bool TimeStretch::SpeechDetection(int32_t vec1_energy, + int32_t vec2_energy, + size_t peak_index, + int scaling) const { + // Check if the signal seems to be active speech or not (simple VAD). + // If (vec1_energy + vec2_energy) / (2 * peak_index) <= + // 8 * background_noise_energy, then we say that the signal contains no + // active speech. + // Rewrite the inequality as: + // (vec1_energy + vec2_energy) / 16 <= peak_index * background_noise_energy. + // The two sides of the inequality will be denoted `left_side` and + // `right_side`. + int32_t left_side = rtc::saturated_cast( + (static_cast(vec1_energy) + vec2_energy) / 16); + int32_t right_side; + if (background_noise_.initialized()) { + right_side = background_noise_.Energy(kRefChannel); + } else { + // If noise parameters have not been estimated, use a fixed threshold. + right_side = 75000; + } + int right_scale = 16 - WebRtcSpl_NormW32(right_side); + right_scale = std::max(0, right_scale); + left_side = left_side >> right_scale; + right_side = + rtc::dchecked_cast(peak_index) * (right_side >> right_scale); + + // Scale `left_side` properly before comparing with `right_side`. + // (`scaling` is the scale factor before energy calculation, thus the scale + // factor for the energy is 2 * scaling.) + if (WebRtcSpl_NormW32(left_side) < 2 * scaling) { + // Cannot scale only `left_side`, must scale `right_side` too. + int temp_scale = WebRtcSpl_NormW32(left_side); + left_side = left_side << temp_scale; + right_side = right_side >> (2 * scaling - temp_scale); + } else { + left_side = left_side << 2 * scaling; + } + return left_side > right_side; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.h b/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.h new file mode 100644 index 0000000000..f0ddaebeca --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TIME_STRETCH_H_ +#define MODULES_AUDIO_CODING_NETEQ_TIME_STRETCH_H_ + +#include // memset, size_t + +#include "modules/audio_coding/neteq/audio_multi_vector.h" + +namespace webrtc { + +// Forward declarations. +class BackgroundNoise; + +// This is the base class for Accelerate and PreemptiveExpand. This class +// cannot be instantiated, but must be used through either of the derived +// classes. +class TimeStretch { + public: + enum ReturnCodes { + kSuccess = 0, + kSuccessLowEnergy = 1, + kNoStretch = 2, + kError = -1 + }; + + TimeStretch(int sample_rate_hz, + size_t num_channels, + const BackgroundNoise& background_noise) + : sample_rate_hz_(sample_rate_hz), + fs_mult_(sample_rate_hz / 8000), + num_channels_(num_channels), + background_noise_(background_noise), + max_input_value_(0) { + RTC_DCHECK(sample_rate_hz_ == 8000 || sample_rate_hz_ == 16000 || + sample_rate_hz_ == 32000 || sample_rate_hz_ == 48000); + RTC_DCHECK_GT(num_channels_, 0); + memset(auto_correlation_, 0, sizeof(auto_correlation_)); + } + + virtual ~TimeStretch() {} + + TimeStretch(const TimeStretch&) = delete; + TimeStretch& operator=(const TimeStretch&) = delete; + + // This method performs the processing common to both Accelerate and + // PreemptiveExpand. + ReturnCodes Process(const int16_t* input, + size_t input_len, + bool fast_mode, + AudioMultiVector* output, + size_t* length_change_samples); + + protected: + // Sets the parameters `best_correlation` and `peak_index` to suitable + // values when the signal contains no active speech. This method must be + // implemented by the sub-classes. + virtual void SetParametersForPassiveSpeech(size_t input_length, + int16_t* best_correlation, + size_t* peak_index) const = 0; + + // Checks the criteria for performing the time-stretching operation and, + // if possible, performs the time-stretching. This method must be implemented + // by the sub-classes. + virtual ReturnCodes CheckCriteriaAndStretch( + const int16_t* input, + size_t input_length, + size_t peak_index, + int16_t best_correlation, + bool active_speech, + bool fast_mode, + AudioMultiVector* output) const = 0; + + static const size_t kCorrelationLen = 50; + static const size_t kLogCorrelationLen = 6; // >= log2(kCorrelationLen). + static const size_t kMinLag = 10; + static const size_t kMaxLag = 60; + static const size_t kDownsampledLen = kCorrelationLen + kMaxLag; + static const int kCorrelationThreshold = 14746; // 0.9 in Q14. + static constexpr size_t kRefChannel = 0; // First channel is reference. + + const int sample_rate_hz_; + const int fs_mult_; // Sample rate multiplier = sample_rate_hz_ / 8000. + const size_t num_channels_; + const BackgroundNoise& background_noise_; + int16_t max_input_value_; + int16_t downsampled_input_[kDownsampledLen]; + // Adding 1 to the size of `auto_correlation_` because of how it is used + // by the peak-detection algorithm. + int16_t auto_correlation_[kCorrelationLen + 1]; + + private: + // Calculates the auto-correlation of `downsampled_input_` and writes the + // result to `auto_correlation_`. + void AutoCorrelation(); + + // Performs a simple voice-activity detection based on the input parameters. + bool SpeechDetection(int32_t vec1_energy, + int32_t vec2_energy, + size_t peak_index, + int scaling) const; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TIME_STRETCH_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch_unittest.cc new file mode 100644 index 0000000000..da3a98229a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch_unittest.cc @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for Accelerate and PreemptiveExpand classes. + +#include +#include + +#include "common_audio/signal_processing/include/signal_processing_library.h" +#include "modules/audio_coding/neteq/accelerate.h" +#include "modules/audio_coding/neteq/background_noise.h" +#include "modules/audio_coding/neteq/preemptive_expand.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +namespace { +const size_t kNumChannels = 1; +} + +TEST(TimeStretch, CreateAndDestroy) { + const int kSampleRate = 8000; + const int kOverlapSamples = 5 * kSampleRate / 8000; + BackgroundNoise bgn(kNumChannels); + Accelerate accelerate(kSampleRate, kNumChannels, bgn); + PreemptiveExpand preemptive_expand(kSampleRate, kNumChannels, bgn, + kOverlapSamples); +} + +TEST(TimeStretch, CreateUsingFactory) { + const int kSampleRate = 8000; + const int kOverlapSamples = 5 * kSampleRate / 8000; + BackgroundNoise bgn(kNumChannels); + + AccelerateFactory accelerate_factory; + Accelerate* accelerate = + accelerate_factory.Create(kSampleRate, kNumChannels, bgn); + EXPECT_TRUE(accelerate != NULL); + delete accelerate; + + PreemptiveExpandFactory preemptive_expand_factory; + PreemptiveExpand* preemptive_expand = preemptive_expand_factory.Create( + kSampleRate, kNumChannels, bgn, kOverlapSamples); + EXPECT_TRUE(preemptive_expand != NULL); + delete preemptive_expand; +} + +class TimeStretchTest : public ::testing::Test { + protected: + TimeStretchTest() + : input_file_(new test::InputAudioFile( + test::ResourcePath("audio_coding/testfile32kHz", "pcm"))), + sample_rate_hz_(32000), + block_size_(30 * sample_rate_hz_ / 1000), // 30 ms + audio_(new int16_t[block_size_]), + background_noise_(kNumChannels) {} + + const int16_t* Next30Ms() { + RTC_CHECK(input_file_->Read(block_size_, audio_.get())); + return audio_.get(); + } + + // Returns the total length change (in samples) that the accelerate operation + // resulted in during the run. + size_t TestAccelerate(size_t loops, bool fast_mode) { + Accelerate accelerate(sample_rate_hz_, kNumChannels, background_noise_); + size_t total_length_change = 0; + for (size_t i = 0; i < loops; ++i) { + AudioMultiVector output(kNumChannels); + size_t length_change; + UpdateReturnStats(accelerate.Process(Next30Ms(), block_size_, fast_mode, + &output, &length_change)); + total_length_change += length_change; + } + return total_length_change; + } + + void UpdateReturnStats(TimeStretch::ReturnCodes ret) { + switch (ret) { + case TimeStretch::kSuccess: + case TimeStretch::kSuccessLowEnergy: + case TimeStretch::kNoStretch: + ++return_stats_[ret]; + break; + case TimeStretch::kError: + FAIL() << "Process returned an error"; + } + } + + std::unique_ptr input_file_; + const int sample_rate_hz_; + const size_t block_size_; + std::unique_ptr audio_; + std::map return_stats_; + BackgroundNoise background_noise_; +}; + +TEST_F(TimeStretchTest, Accelerate) { + // TestAccelerate returns the total length change in samples. + EXPECT_EQ(15268U, TestAccelerate(100, false)); + EXPECT_EQ(9, return_stats_[TimeStretch::kSuccess]); + EXPECT_EQ(58, return_stats_[TimeStretch::kSuccessLowEnergy]); + EXPECT_EQ(33, return_stats_[TimeStretch::kNoStretch]); +} + +TEST_F(TimeStretchTest, AccelerateFastMode) { + // TestAccelerate returns the total length change in samples. + EXPECT_EQ(21400U, TestAccelerate(100, true)); + EXPECT_EQ(31, return_stats_[TimeStretch::kSuccess]); + EXPECT_EQ(58, return_stats_[TimeStretch::kSuccessLowEnergy]); + EXPECT_EQ(11, return_stats_[TimeStretch::kNoStretch]); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.cc b/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.cc new file mode 100644 index 0000000000..59177d027f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/timestamp_scaler.h" + +#include "api/audio_codecs/audio_format.h" +#include "modules/audio_coding/neteq/decoder_database.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +void TimestampScaler::Reset() { + first_packet_received_ = false; +} + +void TimestampScaler::ToInternal(Packet* packet) { + if (!packet) { + return; + } + packet->timestamp = ToInternal(packet->timestamp, packet->payload_type); +} + +void TimestampScaler::ToInternal(PacketList* packet_list) { + PacketList::iterator it; + for (it = packet_list->begin(); it != packet_list->end(); ++it) { + ToInternal(&(*it)); + } +} + +uint32_t TimestampScaler::ToInternal(uint32_t external_timestamp, + uint8_t rtp_payload_type) { + const DecoderDatabase::DecoderInfo* info = + decoder_database_.GetDecoderInfo(rtp_payload_type); + if (!info) { + // Payload type is unknown. Do not scale. + return external_timestamp; + } + if (!(info->IsComfortNoise() || info->IsDtmf())) { + // Do not change the timestamp scaling settings for DTMF or CNG. + numerator_ = info->SampleRateHz(); + if (info->GetFormat().clockrate_hz == 0) { + // If the clockrate is invalid (i.e. with an old-style external codec) + // we cannot do any timestamp scaling. + denominator_ = numerator_; + } else { + denominator_ = info->GetFormat().clockrate_hz; + } + } + if (numerator_ != denominator_) { + // We have a scale factor != 1. + if (!first_packet_received_) { + external_ref_ = external_timestamp; + internal_ref_ = external_timestamp; + first_packet_received_ = true; + } + const int64_t external_diff = int64_t{external_timestamp} - external_ref_; + RTC_DCHECK_GT(denominator_, 0); + external_ref_ = external_timestamp; + internal_ref_ += (external_diff * numerator_) / denominator_; + return internal_ref_; + } else { + // No scaling. + return external_timestamp; + } +} + +uint32_t TimestampScaler::ToExternal(uint32_t internal_timestamp) const { + if (!first_packet_received_ || (numerator_ == denominator_)) { + // Not initialized, or scale factor is 1. + return internal_timestamp; + } else { + const int64_t internal_diff = int64_t{internal_timestamp} - internal_ref_; + RTC_DCHECK_GT(numerator_, 0); + // Do not update references in this method. + // Switch `denominator_` and `numerator_` to convert the other way. + return external_ref_ + (internal_diff * denominator_) / numerator_; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.h b/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.h new file mode 100644 index 0000000000..f42ce7207a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TIMESTAMP_SCALER_H_ +#define MODULES_AUDIO_CODING_NETEQ_TIMESTAMP_SCALER_H_ + +#include "modules/audio_coding/neteq/packet.h" + +namespace webrtc { + +// Forward declaration. +class DecoderDatabase; + +// This class scales timestamps for codecs that need timestamp scaling. +// This is done for codecs where one RTP timestamp does not correspond to +// one sample. +class TimestampScaler { + public: + explicit TimestampScaler(const DecoderDatabase& decoder_database) + : first_packet_received_(false), + numerator_(1), + denominator_(1), + external_ref_(0), + internal_ref_(0), + decoder_database_(decoder_database) {} + + virtual ~TimestampScaler() {} + + TimestampScaler(const TimestampScaler&) = delete; + TimestampScaler& operator=(const TimestampScaler&) = delete; + + // Start over. + virtual void Reset(); + + // Scale the timestamp in `packet` from external to internal. + virtual void ToInternal(Packet* packet); + + // Scale the timestamp for all packets in `packet_list` from external to + // internal. + virtual void ToInternal(PacketList* packet_list); + + // Returns the internal equivalent of `external_timestamp`, given the + // RTP payload type `rtp_payload_type`. + virtual uint32_t ToInternal(uint32_t external_timestamp, + uint8_t rtp_payload_type); + + // Scales back to external timestamp. This is the inverse of ToInternal(). + virtual uint32_t ToExternal(uint32_t internal_timestamp) const; + + private: + bool first_packet_received_; + int numerator_; + int denominator_; + uint32_t external_ref_; + uint32_t internal_ref_; + const DecoderDatabase& decoder_database_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TIMESTAMP_SCALER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler_unittest.cc new file mode 100644 index 0000000000..c2bb4dd95f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler_unittest.cc @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/timestamp_scaler.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/neteq/mock/mock_decoder_database.h" +#include "modules/audio_coding/neteq/packet.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::Return; +using ::testing::ReturnNull; + +namespace webrtc { + +TEST(TimestampScaler, TestNoScaling) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use PCMu, because it doesn't use scaled timestamps. + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadType = 0; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + for (uint32_t timestamp = 0xFFFFFFFF - 5; timestamp != 5; ++timestamp) { + // Scale to internal timestamp. + EXPECT_EQ(timestamp, scaler.ToInternal(timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(timestamp, scaler.ToExternal(timestamp)); + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestNoScalingLargeStep) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use PCMu, because it doesn't use scaled timestamps. + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("pcmu", 8000, 1), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadType = 0; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + static const uint32_t kStep = 160; + uint32_t start_timestamp = 0; + // `external_timestamp` will be a large positive value. + start_timestamp = start_timestamp - 5 * kStep; + for (uint32_t timestamp = start_timestamp; timestamp != 5 * kStep; + timestamp += kStep) { + // Scale to internal timestamp. + EXPECT_EQ(timestamp, scaler.ToInternal(timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(timestamp, scaler.ToExternal(timestamp)); + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestG722) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("g722", 8000, 1), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + for (; external_timestamp != 5; ++external_timestamp) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += 2; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestG722LargeStep) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("g722", 8000, 1), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + static const uint32_t kStep = 320; + uint32_t external_timestamp = 0; + // `external_timestamp` will be a large positive value. + external_timestamp = external_timestamp - 5 * kStep; + uint32_t internal_timestamp = external_timestamp; + for (; external_timestamp != 5 * kStep; external_timestamp += kStep) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + // Internal timestamp should be incremented with twice the step. + internal_timestamp += 2 * kStep; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestG722WithCng) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info_g722(SdpAudioFormat("g722", 8000, 1), + absl::nullopt, factory.get()); + const DecoderDatabase::DecoderInfo info_cng(SdpAudioFormat("cn", 16000, 1), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadTypeG722 = 17; + static const uint8_t kRtpPayloadTypeCng = 13; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadTypeG722)) + .WillRepeatedly(Return(&info_g722)); + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadTypeCng)) + .WillRepeatedly(Return(&info_cng)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + bool next_is_cng = false; + for (; external_timestamp != 5; ++external_timestamp) { + // Alternate between G.722 and CNG every other packet. + if (next_is_cng) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadTypeCng)); + next_is_cng = false; + } else { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadTypeG722)); + next_is_cng = true; + } + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += 2; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +// Make sure that the method ToInternal(Packet* packet) is wired up correctly. +// Since it is simply calling the other ToInternal method, we are not doing +// as many tests here. +TEST(TimestampScaler, TestG722Packet) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("g722", 8000, 1), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + Packet packet; + packet.payload_type = kRtpPayloadType; + for (; external_timestamp != 5; ++external_timestamp) { + packet.timestamp = external_timestamp; + // Scale to internal timestamp. + scaler.ToInternal(&packet); + EXPECT_EQ(internal_timestamp, packet.timestamp); + internal_timestamp += 2; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +// Make sure that the method ToInternal(PacketList* packet_list) is wired up +// correctly. Since it is simply calling the ToInternal(Packet* packet) method, +// we are not doing as many tests here. +TEST(TimestampScaler, TestG722PacketList) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("g722", 8000, 1), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + PacketList packet_list; + { + Packet packet1; + packet1.payload_type = kRtpPayloadType; + packet1.timestamp = external_timestamp; + Packet packet2; + packet2.payload_type = kRtpPayloadType; + packet2.timestamp = external_timestamp + 10; + packet_list.push_back(std::move(packet1)); + packet_list.push_back(std::move(packet2)); + } + + scaler.ToInternal(&packet_list); + EXPECT_EQ(internal_timestamp, packet_list.front().timestamp); + packet_list.pop_front(); + EXPECT_EQ(internal_timestamp + 20, packet_list.front().timestamp); + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, TestG722Reset) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + // Use G722, which has a factor 2 scaling. + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("g722", 8000, 1), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + uint32_t external_timestamp = 0xFFFFFFFF - 5; + uint32_t internal_timestamp = external_timestamp; + for (; external_timestamp != 5; ++external_timestamp) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += 2; + } + // Reset the scaler. After this, we expect the internal and external to start + // over at the same value again. + scaler.Reset(); + internal_timestamp = external_timestamp; + for (; external_timestamp != 15; ++external_timestamp) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += 2; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +// TODO(minyue): This test becomes trivial since Opus does not need a timestamp +// scaler. Therefore, this test may be removed in future. There is no harm to +// keep it, since it can be taken as a test case for the situation of a trivial +// timestamp scaler. +TEST(TimestampScaler, TestOpusLargeStep) { + MockDecoderDatabase db; + auto factory = CreateBuiltinAudioDecoderFactory(); + const DecoderDatabase::DecoderInfo info(SdpAudioFormat("opus", 48000, 2), + absl::nullopt, factory.get()); + static const uint8_t kRtpPayloadType = 17; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillRepeatedly(Return(&info)); + + TimestampScaler scaler(db); + // Test both sides of the timestamp wrap-around. + static const uint32_t kStep = 960; + uint32_t external_timestamp = 0; + // `external_timestamp` will be a large positive value. + external_timestamp = external_timestamp - 5 * kStep; + uint32_t internal_timestamp = external_timestamp; + for (; external_timestamp != 5 * kStep; external_timestamp += kStep) { + // Scale to internal timestamp. + EXPECT_EQ(internal_timestamp, + scaler.ToInternal(external_timestamp, kRtpPayloadType)); + // Scale back. + EXPECT_EQ(external_timestamp, scaler.ToExternal(internal_timestamp)); + internal_timestamp += kStep; + } + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +TEST(TimestampScaler, Failures) { + static const uint8_t kRtpPayloadType = 17; + MockDecoderDatabase db; + EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType)) + .WillOnce(ReturnNull()); // Return NULL to indicate unknown payload type. + + TimestampScaler scaler(db); + uint32_t timestamp = 4711; // Some number. + EXPECT_EQ(timestamp, scaler.ToInternal(timestamp, kRtpPayloadType)); + + Packet* packet = NULL; + scaler.ToInternal(packet); // Should not crash. That's all we can test. + + EXPECT_CALL(db, Die()); // Called when database object is deleted. +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/DEPS b/third_party/libwebrtc/modules/audio_coding/neteq/tools/DEPS new file mode 100644 index 0000000000..4db1e1d6e5 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+logging/rtc_event_log", +] diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/README.md b/third_party/libwebrtc/modules/audio_coding/neteq/tools/README.md new file mode 100644 index 0000000000..e7bd95c285 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/README.md @@ -0,0 +1,17 @@ +# NetEQ RTP Play tool + +## Testing of the command line arguments +The command line tool `neteq_rtpplay` can be tested by running `neteq_rtpplay_test.sh`, which is not use on try bots, but it can be used before submitting any CLs that may break the behavior of the command line arguments of `neteq_rtpplay`. + +Run `neteq_rtpplay_test.sh` as follows from the `src/` folder: +``` +src$ ./modules/audio_coding/neteq/tools/neteq_rtpplay_test.sh \ + out/Default/neteq_rtpplay \ + resources/audio_coding/neteq_opus.rtp \ + resources/short_mixed_mono_48.pcm +``` + +You can replace the RTP and PCM files with any other compatible files. +If you get an error using the files indicated above, try running `gclient sync`. + +Requirements: `awk` and `md5sum`. diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_checksum.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_checksum.h new file mode 100644 index 0000000000..42e3a3a3a0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_checksum.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_CHECKSUM_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_CHECKSUM_H_ + +#include +#include + +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "rtc_base/buffer.h" +#include "rtc_base/message_digest.h" +#include "rtc_base/string_encode.h" +#include "rtc_base/system/arch.h" + +namespace webrtc { +namespace test { + +class AudioChecksum : public AudioSink { + public: + AudioChecksum() + : checksum_(rtc::MessageDigestFactory::Create(rtc::DIGEST_MD5)), + checksum_result_(checksum_->Size()), + finished_(false) {} + + AudioChecksum(const AudioChecksum&) = delete; + AudioChecksum& operator=(const AudioChecksum&) = delete; + + bool WriteArray(const int16_t* audio, size_t num_samples) override { + if (finished_) + return false; + +#ifndef WEBRTC_ARCH_LITTLE_ENDIAN +#error "Big-endian gives a different checksum" +#endif + checksum_->Update(audio, num_samples * sizeof(*audio)); + return true; + } + + // Finalizes the computations, and returns the checksum. + std::string Finish() { + if (!finished_) { + finished_ = true; + checksum_->Finish(checksum_result_.data(), checksum_result_.size()); + } + return rtc::hex_encode(checksum_result_); + } + + private: + std::unique_ptr checksum_; + rtc::Buffer checksum_result_; + bool finished_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_CHECKSUM_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_loop.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_loop.cc new file mode 100644 index 0000000000..514e6eb2ee --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_loop.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/audio_loop.h" + +#include +#include + +#include "absl/strings/string_view.h" + +namespace webrtc { +namespace test { + +bool AudioLoop::Init(absl::string_view file_name, + size_t max_loop_length_samples, + size_t block_length_samples) { + FILE* fp = fopen(std::string(file_name).c_str(), "rb"); + if (!fp) + return false; + + audio_array_.reset( + new int16_t[max_loop_length_samples + block_length_samples]); + size_t samples_read = + fread(audio_array_.get(), sizeof(int16_t), max_loop_length_samples, fp); + fclose(fp); + + // Block length must be shorter than the loop length. + if (block_length_samples > samples_read) + return false; + + // Add an extra block length of samples to the end of the array, starting + // over again from the beginning of the array. This is done to simplify + // the reading process when reading over the end of the loop. + memcpy(&audio_array_[samples_read], audio_array_.get(), + block_length_samples * sizeof(int16_t)); + + loop_length_samples_ = samples_read; + block_length_samples_ = block_length_samples; + next_index_ = 0; + return true; +} + +rtc::ArrayView AudioLoop::GetNextBlock() { + // Check that the AudioLoop is initialized. + if (block_length_samples_ == 0) + return rtc::ArrayView(); + + const int16_t* output_ptr = &audio_array_[next_index_]; + next_index_ = (next_index_ + block_length_samples_) % loop_length_samples_; + return rtc::ArrayView(output_ptr, block_length_samples_); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_loop.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_loop.h new file mode 100644 index 0000000000..f5f0b59011 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_loop.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_LOOP_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_LOOP_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "api/array_view.h" + +namespace webrtc { +namespace test { + +// Class serving as an infinite source of audio, realized by looping an audio +// clip. +class AudioLoop { + public: + AudioLoop() + : next_index_(0), loop_length_samples_(0), block_length_samples_(0) {} + + virtual ~AudioLoop() {} + + AudioLoop(const AudioLoop&) = delete; + AudioLoop& operator=(const AudioLoop&) = delete; + + // Initializes the AudioLoop by reading from `file_name`. The loop will be no + // longer than `max_loop_length_samples`, if the length of the file is + // greater. Otherwise, the loop length is the same as the file length. + // The audio will be delivered in blocks of `block_length_samples`. + // Returns false if the initialization failed, otherwise true. + bool Init(absl::string_view file_name, + size_t max_loop_length_samples, + size_t block_length_samples); + + // Returns a (pointer,size) pair for the next block of audio. The size is + // equal to the `block_length_samples` Init() argument. + rtc::ArrayView GetNextBlock(); + + private: + size_t next_index_; + size_t loop_length_samples_; + size_t block_length_samples_; + std::unique_ptr audio_array_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_LOOP_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_sink.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_sink.cc new file mode 100644 index 0000000000..7d7af7ef9f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_sink.cc @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/audio_sink.h" + +namespace webrtc { +namespace test { + +bool AudioSinkFork::WriteArray(const int16_t* audio, size_t num_samples) { + return left_sink_->WriteArray(audio, num_samples) && + right_sink_->WriteArray(audio, num_samples); +} + +bool VoidAudioSink::WriteArray(const int16_t* audio, size_t num_samples) { + return true; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_sink.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_sink.h new file mode 100644 index 0000000000..53729fa920 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/audio_sink.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_SINK_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_SINK_H_ + +#include "api/audio/audio_frame.h" + +namespace webrtc { +namespace test { + +// Interface class for an object receiving raw output audio from test +// applications. +class AudioSink { + public: + AudioSink() {} + virtual ~AudioSink() {} + + AudioSink(const AudioSink&) = delete; + AudioSink& operator=(const AudioSink&) = delete; + + // Writes `num_samples` from `audio` to the AudioSink. Returns true if + // successful, otherwise false. + virtual bool WriteArray(const int16_t* audio, size_t num_samples) = 0; + + // Writes `audio_frame` to the AudioSink. Returns true if successful, + // otherwise false. + bool WriteAudioFrame(const AudioFrame& audio_frame) { + return WriteArray(audio_frame.data(), audio_frame.samples_per_channel_ * + audio_frame.num_channels_); + } +}; + +// Forks the output audio to two AudioSink objects. +class AudioSinkFork : public AudioSink { + public: + AudioSinkFork(AudioSink* left, AudioSink* right) + : left_sink_(left), right_sink_(right) {} + + AudioSinkFork(const AudioSinkFork&) = delete; + AudioSinkFork& operator=(const AudioSinkFork&) = delete; + + bool WriteArray(const int16_t* audio, size_t num_samples) override; + + private: + AudioSink* left_sink_; + AudioSink* right_sink_; +}; + +// An AudioSink implementation that does nothing. +class VoidAudioSink : public AudioSink { + public: + VoidAudioSink() = default; + + VoidAudioSink(const VoidAudioSink&) = delete; + VoidAudioSink& operator=(const VoidAudioSink&) = delete; + + bool WriteArray(const int16_t* audio, size_t num_samples) override; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_SINK_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.cc new file mode 100644 index 0000000000..18a910365f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/constant_pcm_packet_source.h" + +#include + +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +ConstantPcmPacketSource::ConstantPcmPacketSource(size_t payload_len_samples, + int16_t sample_value, + int sample_rate_hz, + int payload_type) + : payload_len_samples_(payload_len_samples), + packet_len_bytes_(2 * payload_len_samples_ + kHeaderLenBytes), + samples_per_ms_(sample_rate_hz / 1000), + next_arrival_time_ms_(0.0), + payload_type_(payload_type), + seq_number_(0), + timestamp_(0), + payload_ssrc_(0xABCD1234) { + size_t encoded_len = WebRtcPcm16b_Encode(&sample_value, 1, encoded_sample_); + RTC_CHECK_EQ(2U, encoded_len); +} + +std::unique_ptr ConstantPcmPacketSource::NextPacket() { + RTC_CHECK_GT(packet_len_bytes_, kHeaderLenBytes); + rtc::CopyOnWriteBuffer packet_buffer(packet_len_bytes_); + uint8_t* packet_memory = packet_buffer.MutableData(); + // Fill the payload part of the packet memory with the pre-encoded value. + for (unsigned i = 0; i < 2 * payload_len_samples_; ++i) + packet_memory[kHeaderLenBytes + i] = encoded_sample_[i % 2]; + WriteHeader(packet_memory); + // `packet` assumes ownership of `packet_memory`. + auto packet = + std::make_unique(std::move(packet_buffer), next_arrival_time_ms_); + next_arrival_time_ms_ += payload_len_samples_ / samples_per_ms_; + return packet; +} + +void ConstantPcmPacketSource::WriteHeader(uint8_t* packet_memory) { + packet_memory[0] = 0x80; + packet_memory[1] = static_cast(payload_type_); + packet_memory[2] = seq_number_ >> 8; + packet_memory[3] = seq_number_ & 0xFF; + packet_memory[4] = timestamp_ >> 24; + packet_memory[5] = (timestamp_ >> 16) & 0xFF; + packet_memory[6] = (timestamp_ >> 8) & 0xFF; + packet_memory[7] = timestamp_ & 0xFF; + packet_memory[8] = payload_ssrc_ >> 24; + packet_memory[9] = (payload_ssrc_ >> 16) & 0xFF; + packet_memory[10] = (payload_ssrc_ >> 8) & 0xFF; + packet_memory[11] = payload_ssrc_ & 0xFF; + ++seq_number_; + timestamp_ += static_cast(payload_len_samples_); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.h new file mode 100644 index 0000000000..ab4f5c2281 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/constant_pcm_packet_source.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_CONSTANT_PCM_PACKET_SOURCE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_CONSTANT_PCM_PACKET_SOURCE_H_ + +#include + +#include + +#include "modules/audio_coding/neteq/tools/packet_source.h" + +namespace webrtc { +namespace test { + +// This class implements a packet source that delivers PCM16b encoded packets +// with a constant sample value. The payload length, constant sample value, +// sample rate, and payload type are all set in the constructor. +class ConstantPcmPacketSource : public PacketSource { + public: + ConstantPcmPacketSource(size_t payload_len_samples, + int16_t sample_value, + int sample_rate_hz, + int payload_type); + + ConstantPcmPacketSource(const ConstantPcmPacketSource&) = delete; + ConstantPcmPacketSource& operator=(const ConstantPcmPacketSource&) = delete; + + std::unique_ptr NextPacket() override; + + private: + void WriteHeader(uint8_t* packet_memory); + + const size_t kHeaderLenBytes = 12; + const size_t payload_len_samples_; + const size_t packet_len_bytes_; + uint8_t encoded_sample_[2]; + const int samples_per_ms_; + double next_arrival_time_ms_; + const int payload_type_; + uint16_t seq_number_; + uint32_t timestamp_; + const uint32_t payload_ssrc_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_CONSTANT_PCM_PACKET_SOURCE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/encode_neteq_input.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/encode_neteq_input.cc new file mode 100644 index 0000000000..87b987ddb6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/encode_neteq_input.cc @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/encode_neteq_input.h" + +#include + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { +namespace test { + +EncodeNetEqInput::EncodeNetEqInput(std::unique_ptr generator, + std::unique_ptr encoder, + int64_t input_duration_ms) + : generator_(std::move(generator)), + encoder_(std::move(encoder)), + input_duration_ms_(input_duration_ms) { + CreatePacket(); +} + +EncodeNetEqInput::~EncodeNetEqInput() = default; + +absl::optional EncodeNetEqInput::NextPacketTime() const { + RTC_DCHECK(packet_data_); + return static_cast(packet_data_->time_ms); +} + +absl::optional EncodeNetEqInput::NextOutputEventTime() const { + return next_output_event_ms_; +} + +std::unique_ptr EncodeNetEqInput::PopPacket() { + RTC_DCHECK(packet_data_); + // Grab the packet to return... + std::unique_ptr packet_to_return = std::move(packet_data_); + // ... and line up the next packet for future use. + CreatePacket(); + + return packet_to_return; +} + +void EncodeNetEqInput::AdvanceOutputEvent() { + next_output_event_ms_ += kOutputPeriodMs; +} + +bool EncodeNetEqInput::ended() const { + return next_output_event_ms_ > input_duration_ms_; +} + +absl::optional EncodeNetEqInput::NextHeader() const { + RTC_DCHECK(packet_data_); + return packet_data_->header; +} + +void EncodeNetEqInput::CreatePacket() { + // Create a new PacketData object. + RTC_DCHECK(!packet_data_); + packet_data_.reset(new NetEqInput::PacketData); + RTC_DCHECK_EQ(packet_data_->payload.size(), 0); + + // Loop until we get a packet. + AudioEncoder::EncodedInfo info; + RTC_DCHECK(!info.send_even_if_empty); + int num_blocks = 0; + while (packet_data_->payload.size() == 0 && !info.send_even_if_empty) { + const size_t num_samples = rtc::CheckedDivExact( + static_cast(encoder_->SampleRateHz() * kOutputPeriodMs), 1000); + + info = encoder_->Encode(rtp_timestamp_, generator_->Generate(num_samples), + &packet_data_->payload); + + rtp_timestamp_ += rtc::dchecked_cast( + num_samples * encoder_->RtpTimestampRateHz() / + encoder_->SampleRateHz()); + ++num_blocks; + } + packet_data_->header.timestamp = info.encoded_timestamp; + packet_data_->header.payloadType = info.payload_type; + packet_data_->header.sequenceNumber = sequence_number_++; + packet_data_->time_ms = next_packet_time_ms_; + next_packet_time_ms_ += num_blocks * kOutputPeriodMs; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/encode_neteq_input.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/encode_neteq_input.h new file mode 100644 index 0000000000..caa9ac76f4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/encode_neteq_input.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_ENCODE_NETEQ_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_ENCODE_NETEQ_INPUT_H_ + +#include + +#include "api/audio_codecs/audio_encoder.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" + +namespace webrtc { +namespace test { + +// This class provides a NetEqInput that takes audio from a generator object and +// encodes it using a given audio encoder. +class EncodeNetEqInput : public NetEqInput { + public: + // Generator class, to be provided to the EncodeNetEqInput constructor. + class Generator { + public: + virtual ~Generator() = default; + // Returns the next num_samples values from the signal generator. + virtual rtc::ArrayView Generate(size_t num_samples) = 0; + }; + + // The source will end after the given input duration. + EncodeNetEqInput(std::unique_ptr generator, + std::unique_ptr encoder, + int64_t input_duration_ms); + ~EncodeNetEqInput() override; + + absl::optional NextPacketTime() const override; + + absl::optional NextOutputEventTime() const override; + + std::unique_ptr PopPacket() override; + + void AdvanceOutputEvent() override; + + bool ended() const override; + + absl::optional NextHeader() const override; + + private: + static constexpr int64_t kOutputPeriodMs = 10; + + void CreatePacket(); + + std::unique_ptr generator_; + std::unique_ptr encoder_; + std::unique_ptr packet_data_; + uint32_t rtp_timestamp_ = 0; + int16_t sequence_number_ = 0; + int64_t next_packet_time_ms_ = 0; + int64_t next_output_event_ms_ = 0; + const int64_t input_duration_ms_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_ENCODE_NETEQ_INPUT_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.cc new file mode 100644 index 0000000000..6c5e5ac2e4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.cc @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/fake_decode_from_file.h" + +#include "modules/rtp_rtcp/source/byte_io.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { +namespace test { + +namespace { + +class FakeEncodedFrame : public AudioDecoder::EncodedAudioFrame { + public: + FakeEncodedFrame(AudioDecoder* decoder, rtc::Buffer&& payload) + : decoder_(decoder), payload_(std::move(payload)) {} + + size_t Duration() const override { + const int ret = decoder_->PacketDuration(payload_.data(), payload_.size()); + return ret < 0 ? 0 : static_cast(ret); + } + + absl::optional Decode( + rtc::ArrayView decoded) const override { + auto speech_type = AudioDecoder::kSpeech; + const int ret = decoder_->Decode( + payload_.data(), payload_.size(), decoder_->SampleRateHz(), + decoded.size() * sizeof(int16_t), decoded.data(), &speech_type); + return ret < 0 ? absl::nullopt + : absl::optional( + {static_cast(ret), speech_type}); + } + + // This is to mimic OpusFrame. + bool IsDtxPacket() const override { + uint32_t original_payload_size_bytes = + ByteReader::ReadLittleEndian(&payload_.data()[8]); + return original_payload_size_bytes <= 2; + } + + private: + AudioDecoder* const decoder_; + const rtc::Buffer payload_; +}; + +} // namespace + +std::vector FakeDecodeFromFile::ParsePayload( + rtc::Buffer&& payload, + uint32_t timestamp) { + std::vector results; + std::unique_ptr frame( + new FakeEncodedFrame(this, std::move(payload))); + results.emplace_back(timestamp, 0, std::move(frame)); + return results; +} + +int FakeDecodeFromFile::DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) { + RTC_DCHECK_EQ(sample_rate_hz, SampleRateHz()); + + const int samples_to_decode = PacketDuration(encoded, encoded_len); + const int total_samples_to_decode = samples_to_decode * (stereo_ ? 2 : 1); + + if (encoded_len == 0) { + // Decoder is asked to produce codec-internal comfort noise. + RTC_DCHECK(!encoded); // NetEq always sends nullptr in this case. + RTC_DCHECK(cng_mode_); + RTC_DCHECK_GT(total_samples_to_decode, 0); + std::fill_n(decoded, total_samples_to_decode, 0); + *speech_type = kComfortNoise; + return rtc::dchecked_cast(total_samples_to_decode); + } + + RTC_CHECK_GE(encoded_len, 12); + uint32_t timestamp_to_decode = + ByteReader::ReadLittleEndian(encoded); + + if (next_timestamp_from_input_ && + timestamp_to_decode != *next_timestamp_from_input_) { + // A gap in the timestamp sequence is detected. Skip the same number of + // samples from the file. + uint32_t jump = timestamp_to_decode - *next_timestamp_from_input_; + RTC_CHECK(input_->Seek(jump)); + } + + next_timestamp_from_input_ = timestamp_to_decode + samples_to_decode; + + uint32_t original_payload_size_bytes = + ByteReader::ReadLittleEndian(&encoded[8]); + if (original_payload_size_bytes <= 2) { + // This is a comfort noise payload. + RTC_DCHECK_GT(total_samples_to_decode, 0); + std::fill_n(decoded, total_samples_to_decode, 0); + *speech_type = kComfortNoise; + cng_mode_ = true; + return rtc::dchecked_cast(total_samples_to_decode); + } + + cng_mode_ = false; + RTC_CHECK(input_->Read(static_cast(samples_to_decode), decoded)); + + if (stereo_) { + InputAudioFile::DuplicateInterleaved(decoded, samples_to_decode, 2, + decoded); + } + + *speech_type = kSpeech; + last_decoded_length_ = samples_to_decode; + return rtc::dchecked_cast(total_samples_to_decode); +} + +int FakeDecodeFromFile::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { + const uint32_t original_payload_size_bytes = + encoded_len < 8 + sizeof(uint32_t) + ? 0 + : ByteReader::ReadLittleEndian(&encoded[8]); + const uint32_t samples_to_decode = + encoded_len < 4 + sizeof(uint32_t) + ? 0 + : ByteReader::ReadLittleEndian(&encoded[4]); + if (encoded_len == 0) { + // Decoder is asked to produce codec-internal comfort noise. + return rtc::CheckedDivExact(SampleRateHz(), 100); + } + bool is_dtx_payload = + original_payload_size_bytes <= 2 || samples_to_decode == 0; + bool has_error_duration = + samples_to_decode % rtc::CheckedDivExact(SampleRateHz(), 100) != 0; + if (is_dtx_payload || has_error_duration) { + if (last_decoded_length_ > 0) { + // Use length of last decoded packet. + return rtc::dchecked_cast(last_decoded_length_); + } else { + // This is the first packet to decode, and we do not know the length of + // it. Set it to 10 ms. + return rtc::CheckedDivExact(SampleRateHz(), 100); + } + } + return samples_to_decode; +} + +void FakeDecodeFromFile::PrepareEncoded(uint32_t timestamp, + size_t samples, + size_t original_payload_size_bytes, + rtc::ArrayView encoded) { + RTC_CHECK_GE(encoded.size(), 12); + ByteWriter::WriteLittleEndian(&encoded[0], timestamp); + ByteWriter::WriteLittleEndian(&encoded[4], + rtc::checked_cast(samples)); + ByteWriter::WriteLittleEndian( + &encoded[8], rtc::checked_cast(original_payload_size_bytes)); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h new file mode 100644 index 0000000000..7b53653998 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_FAKE_DECODE_FROM_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_FAKE_DECODE_FROM_FILE_H_ + +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/audio_codecs/audio_decoder.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" + +namespace webrtc { +namespace test { +// Provides an AudioDecoder implementation that delivers audio data from a file. +// The "encoded" input should contain information about what RTP timestamp the +// encoding represents, and how many samples the decoder should produce for that +// encoding. A helper method PrepareEncoded is provided to prepare such +// encodings. If packets are missing, as determined from the timestamps, the +// file reading will skip forward to match the loss. +class FakeDecodeFromFile : public AudioDecoder { + public: + FakeDecodeFromFile(std::unique_ptr input, + int sample_rate_hz, + bool stereo) + : input_(std::move(input)), + sample_rate_hz_(sample_rate_hz), + stereo_(stereo) {} + + ~FakeDecodeFromFile() = default; + + std::vector ParsePayload(rtc::Buffer&& payload, + uint32_t timestamp) override; + + void Reset() override {} + + int SampleRateHz() const override { return sample_rate_hz_; } + + size_t Channels() const override { return stereo_ ? 2 : 1; } + + int DecodeInternal(const uint8_t* encoded, + size_t encoded_len, + int sample_rate_hz, + int16_t* decoded, + SpeechType* speech_type) override; + + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; + + // Helper method. Writes `timestamp`, `samples` and + // `original_payload_size_bytes` to `encoded` in a format that the + // FakeDecodeFromFile decoder will understand. `encoded` must be at least 12 + // bytes long. + static void PrepareEncoded(uint32_t timestamp, + size_t samples, + size_t original_payload_size_bytes, + rtc::ArrayView encoded); + + private: + std::unique_ptr input_; + absl::optional next_timestamp_from_input_; + const int sample_rate_hz_; + const bool stereo_; + size_t last_decoded_length_ = 0; + bool cng_mode_ = false; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_FAKE_DECODE_FROM_FILE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc new file mode 100644 index 0000000000..3c33aabf1c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h" + +#include +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +InitialPacketInserterNetEqInput::InitialPacketInserterNetEqInput( + std::unique_ptr source, + int number_of_initial_packets, + int sample_rate_hz) + : source_(std::move(source)), + packets_to_insert_(number_of_initial_packets), + sample_rate_hz_(sample_rate_hz) {} + +absl::optional InitialPacketInserterNetEqInput::NextPacketTime() + const { + return source_->NextPacketTime(); +} + +absl::optional InitialPacketInserterNetEqInput::NextOutputEventTime() + const { + return source_->NextOutputEventTime(); +} + +std::unique_ptr +InitialPacketInserterNetEqInput::PopPacket() { + if (!first_packet_) { + first_packet_ = source_->PopPacket(); + if (!first_packet_) { + // The source has no packets, so we should not insert any dummy packets. + packets_to_insert_ = 0; + } + } + if (packets_to_insert_ > 0) { + RTC_CHECK(first_packet_); + auto dummy_packet = std::unique_ptr(new PacketData()); + dummy_packet->header = first_packet_->header; + dummy_packet->payload = rtc::Buffer(first_packet_->payload.data(), + first_packet_->payload.size()); + dummy_packet->time_ms = first_packet_->time_ms; + dummy_packet->header.sequenceNumber -= packets_to_insert_; + // This assumes 20ms per packet. + dummy_packet->header.timestamp -= + 20 * sample_rate_hz_ * packets_to_insert_ / 1000; + packets_to_insert_--; + return dummy_packet; + } + return source_->PopPacket(); +} + +void InitialPacketInserterNetEqInput::AdvanceOutputEvent() { + source_->AdvanceOutputEvent(); +} + +bool InitialPacketInserterNetEqInput::ended() const { + return source_->ended(); +} + +absl::optional InitialPacketInserterNetEqInput::NextHeader() const { + return source_->NextHeader(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h new file mode 100644 index 0000000000..bd20a7aecf --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_INITIAL_PACKET_INSERTER_NETEQ_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_INITIAL_PACKET_INSERTER_NETEQ_INPUT_H_ + +#include +#include +#include + +#include "modules/audio_coding/neteq/tools/neteq_input.h" + +namespace webrtc { +namespace test { + +// Wrapper class that can insert a number of packets at the start of the +// simulation. +class InitialPacketInserterNetEqInput final : public NetEqInput { + public: + InitialPacketInserterNetEqInput(std::unique_ptr source, + int number_of_initial_packets, + int sample_rate_hz); + absl::optional NextPacketTime() const override; + absl::optional NextOutputEventTime() const override; + std::unique_ptr PopPacket() override; + void AdvanceOutputEvent() override; + bool ended() const override; + absl::optional NextHeader() const override; + + private: + const std::unique_ptr source_; + int packets_to_insert_; + const int sample_rate_hz_; + std::unique_ptr first_packet_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_INITIAL_PACKET_INSERTER_NETEQ_INPUT_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file.cc new file mode 100644 index 0000000000..b077dbff21 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/input_audio_file.h" + +#include "absl/strings/string_view.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +InputAudioFile::InputAudioFile(absl::string_view file_name, bool loop_at_end) + : loop_at_end_(loop_at_end) { + fp_ = fopen(std::string(file_name).c_str(), "rb"); + RTC_DCHECK(fp_) << file_name << " could not be opened."; +} + +InputAudioFile::~InputAudioFile() { + RTC_DCHECK(fp_); + fclose(fp_); +} + +bool InputAudioFile::Read(size_t samples, int16_t* destination) { + if (!fp_) { + return false; + } + size_t samples_read = fread(destination, sizeof(int16_t), samples, fp_); + if (samples_read < samples) { + if (!loop_at_end_) { + return false; + } + // Rewind and read the missing samples. + rewind(fp_); + size_t missing_samples = samples - samples_read; + if (fread(destination + samples_read, sizeof(int16_t), missing_samples, + fp_) < missing_samples) { + // Could not read enough even after rewinding the file. + return false; + } + } + return true; +} + +bool InputAudioFile::Seek(int samples) { + if (!fp_) { + return false; + } + // Find file boundaries. + const long current_pos = ftell(fp_); + RTC_CHECK_NE(EOF, current_pos) + << "Error returned when getting file position."; + RTC_CHECK_EQ(0, fseek(fp_, 0, SEEK_END)); // Move to end of file. + const long file_size = ftell(fp_); + RTC_CHECK_NE(EOF, file_size) << "Error returned when getting file position."; + // Find new position. + long new_pos = current_pos + sizeof(int16_t) * samples; // Samples to bytes. + if (loop_at_end_) { + new_pos = new_pos % file_size; // Wrap around the end of the file. + if (new_pos < 0) { + // For negative values of new_pos, newpos % file_size will also be + // negative. To get the correct result it's needed to add file_size. + new_pos += file_size; + } + } else { + new_pos = new_pos > file_size ? file_size : new_pos; // Don't loop. + } + RTC_CHECK_GE(new_pos, 0) + << "Trying to move to before the beginning of the file"; + // Move to new position relative to the beginning of the file. + RTC_CHECK_EQ(0, fseek(fp_, new_pos, SEEK_SET)); + return true; +} + +void InputAudioFile::DuplicateInterleaved(const int16_t* source, + size_t samples, + size_t channels, + int16_t* destination) { + // Start from the end of `source` and `destination`, and work towards the + // beginning. This is to allow in-place interleaving of the same array (i.e., + // `source` and `destination` are the same array). + for (int i = static_cast(samples - 1); i >= 0; --i) { + for (int j = static_cast(channels - 1); j >= 0; --j) { + destination[i * channels + j] = source[i]; + } + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file.h new file mode 100644 index 0000000000..f538b295a3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_INPUT_AUDIO_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_INPUT_AUDIO_FILE_H_ + +#include + +#include + +#include "absl/strings/string_view.h" + +namespace webrtc { +namespace test { + +// Class for handling a looping input audio file. +class InputAudioFile { + public: + explicit InputAudioFile(absl::string_view file_name, bool loop_at_end = true); + + virtual ~InputAudioFile(); + + InputAudioFile(const InputAudioFile&) = delete; + InputAudioFile& operator=(const InputAudioFile&) = delete; + + // Reads `samples` elements from source file to `destination`. Returns true + // if the read was successful, otherwise false. If the file end is reached, + // the file is rewound and reading continues from the beginning. + // The output `destination` must have the capacity to hold `samples` elements. + virtual bool Read(size_t samples, int16_t* destination); + + // Fast-forwards (`samples` > 0) or -backwards (`samples` < 0) the file by the + // indicated number of samples. Just like Read(), Seek() starts over at the + // beginning of the file if the end is reached. However, seeking backwards + // past the beginning of the file is not possible. + virtual bool Seek(int samples); + + // Creates a multi-channel signal from a mono signal. Each sample is repeated + // `channels` times to create an interleaved multi-channel signal where all + // channels are identical. The output `destination` must have the capacity to + // hold samples * channels elements. Note that `source` and `destination` can + // be the same array (i.e., point to the same address). + static void DuplicateInterleaved(const int16_t* source, + size_t samples, + size_t channels, + int16_t* destination); + + private: + FILE* fp_; + const bool loop_at_end_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_INPUT_AUDIO_FILE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file_unittest.cc new file mode 100644 index 0000000000..52f7ea82a0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/input_audio_file_unittest.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for test InputAudioFile class. + +#include "modules/audio_coding/neteq/tools/input_audio_file.h" + +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +TEST(TestInputAudioFile, DuplicateInterleaveSeparateSrcDst) { + static const size_t kSamples = 10; + static const size_t kChannels = 2; + int16_t input[kSamples]; + for (size_t i = 0; i < kSamples; ++i) { + input[i] = rtc::checked_cast(i); + } + int16_t output[kSamples * kChannels]; + InputAudioFile::DuplicateInterleaved(input, kSamples, kChannels, output); + + // Verify output + int16_t* output_ptr = output; + for (size_t i = 0; i < kSamples; ++i) { + for (size_t j = 0; j < kChannels; ++j) { + EXPECT_EQ(static_cast(i), *output_ptr++); + } + } +} + +TEST(TestInputAudioFile, DuplicateInterleaveSameSrcDst) { + static const size_t kSamples = 10; + static const size_t kChannels = 5; + int16_t input[kSamples * kChannels]; + for (size_t i = 0; i < kSamples; ++i) { + input[i] = rtc::checked_cast(i); + } + InputAudioFile::DuplicateInterleaved(input, kSamples, kChannels, input); + + // Verify output + int16_t* output_ptr = input; + for (size_t i = 0; i < kSamples; ++i) { + for (size_t j = 0; j < kChannels; ++j) { + EXPECT_EQ(static_cast(i), *output_ptr++); + } + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc new file mode 100644 index 0000000000..9e77457775 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" + +namespace webrtc { +namespace test { +namespace { +constexpr char kArrivalDelayX[] = "arrival_delay_x"; +constexpr char kArrivalDelayY[] = "arrival_delay_y"; +constexpr char kTargetDelayX[] = "target_delay_x"; +constexpr char kTargetDelayY[] = "target_delay_y"; +constexpr char kPlayoutDelayX[] = "playout_delay_x"; +constexpr char kPlayoutDelayY[] = "playout_delay_y"; + +// Helper function for NetEqDelayAnalyzer::CreateGraphs. Returns the +// interpolated value of a function at the point x. Vector x_vec contains the +// sample points, and y_vec contains the function values at these points. The +// return value is a linear interpolation between y_vec values. +double LinearInterpolate(double x, + const std::vector& x_vec, + const std::vector& y_vec) { + // Find first element which is larger than x. + auto it = std::upper_bound(x_vec.begin(), x_vec.end(), x); + if (it == x_vec.end()) { + --it; + } + const size_t upper_ix = it - x_vec.begin(); + + size_t lower_ix; + if (upper_ix == 0 || x_vec[upper_ix] <= x) { + lower_ix = upper_ix; + } else { + lower_ix = upper_ix - 1; + } + double y; + if (lower_ix == upper_ix) { + y = y_vec[lower_ix]; + } else { + RTC_DCHECK_NE(x_vec[lower_ix], x_vec[upper_ix]); + y = (x - x_vec[lower_ix]) * (y_vec[upper_ix] - y_vec[lower_ix]) / + (x_vec[upper_ix] - x_vec[lower_ix]) + + y_vec[lower_ix]; + } + return y; +} + +void PrintDelays(const NetEqDelayAnalyzer::Delays& delays, + int64_t ref_time_ms, + absl::string_view var_name_x, + absl::string_view var_name_y, + std::ofstream& output, + absl::string_view terminator = "") { + output << var_name_x << " = [ "; + for (const std::pair& delay : delays) { + output << (delay.first - ref_time_ms) / 1000.f << ", "; + } + output << "]" << terminator << std::endl; + + output << var_name_y << " = [ "; + for (const std::pair& delay : delays) { + output << delay.second << ", "; + } + output << "]" << terminator << std::endl; +} + +} // namespace + +void NetEqDelayAnalyzer::AfterInsertPacket( + const test::NetEqInput::PacketData& packet, + NetEq* neteq) { + data_.insert( + std::make_pair(packet.header.timestamp, TimingData(packet.time_ms))); + ssrcs_.insert(packet.header.ssrc); + payload_types_.insert(packet.header.payloadType); +} + +void NetEqDelayAnalyzer::BeforeGetAudio(NetEq* neteq) { + last_sync_buffer_ms_ = neteq->SyncBufferSizeMs(); +} + +void NetEqDelayAnalyzer::AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool /*muted*/, + NetEq* neteq) { + get_audio_time_ms_.push_back(time_now_ms); + for (const RtpPacketInfo& info : audio_frame.packet_infos_) { + auto it = data_.find(info.rtp_timestamp()); + if (it == data_.end()) { + // This is a packet that was split out from another packet. Skip it. + continue; + } + auto& it_timing = it->second; + RTC_CHECK(!it_timing.decode_get_audio_count) + << "Decode time already written"; + it_timing.decode_get_audio_count = get_audio_count_; + RTC_CHECK(!it_timing.sync_delay_ms) << "Decode time already written"; + it_timing.sync_delay_ms = last_sync_buffer_ms_; + it_timing.target_delay_ms = neteq->TargetDelayMs(); + it_timing.current_delay_ms = neteq->FilteredCurrentDelayMs(); + } + last_sample_rate_hz_ = audio_frame.sample_rate_hz_; + ++get_audio_count_; +} + +void NetEqDelayAnalyzer::CreateGraphs(Delays* arrival_delay_ms, + Delays* corrected_arrival_delay_ms, + Delays* playout_delay_ms, + Delays* target_delay_ms) const { + if (get_audio_time_ms_.empty()) { + return; + } + // Create nominal_get_audio_time_ms, a vector starting at + // get_audio_time_ms_[0] and increasing by 10 for each element. + std::vector nominal_get_audio_time_ms(get_audio_time_ms_.size()); + nominal_get_audio_time_ms[0] = get_audio_time_ms_[0]; + std::transform( + nominal_get_audio_time_ms.begin(), nominal_get_audio_time_ms.end() - 1, + nominal_get_audio_time_ms.begin() + 1, [](int64_t& x) { return x + 10; }); + RTC_DCHECK( + std::is_sorted(get_audio_time_ms_.begin(), get_audio_time_ms_.end())); + + std::vector rtp_timestamps_ms; + double offset = std::numeric_limits::max(); + RtpTimestampUnwrapper unwrapper; + // This loop traverses data_ and populates rtp_timestamps_ms as well as + // calculates the base offset. + for (auto& d : data_) { + rtp_timestamps_ms.push_back( + static_cast(unwrapper.Unwrap(d.first)) / + rtc::CheckedDivExact(last_sample_rate_hz_, 1000)); + offset = + std::min(offset, d.second.arrival_time_ms - rtp_timestamps_ms.back()); + } + + // This loop traverses the data again and populates the graph vectors. The + // reason to have two loops and traverse twice is that the offset cannot be + // known until the first traversal is done. Meanwhile, the final offset must + // be known already at the start of this second loop. + size_t i = 0; + for (const auto& data : data_) { + const double offset_send_time_ms = rtp_timestamps_ms[i++] + offset; + const auto& timing = data.second; + corrected_arrival_delay_ms->push_back(std::make_pair( + timing.arrival_time_ms, + LinearInterpolate(timing.arrival_time_ms, get_audio_time_ms_, + nominal_get_audio_time_ms) - + offset_send_time_ms)); + arrival_delay_ms->push_back(std::make_pair( + timing.arrival_time_ms, timing.arrival_time_ms - offset_send_time_ms)); + + if (timing.decode_get_audio_count) { + // This packet was decoded. + RTC_DCHECK(timing.sync_delay_ms); + const int64_t get_audio_time = + *timing.decode_get_audio_count * 10 + get_audio_time_ms_[0]; + const float playout_ms = + get_audio_time + *timing.sync_delay_ms - offset_send_time_ms; + playout_delay_ms->push_back(std::make_pair(get_audio_time, playout_ms)); + RTC_DCHECK(timing.target_delay_ms); + RTC_DCHECK(timing.current_delay_ms); + const float target = + playout_ms - *timing.current_delay_ms + *timing.target_delay_ms; + target_delay_ms->push_back(std::make_pair(get_audio_time, target)); + } + } +} + +void NetEqDelayAnalyzer::CreateMatlabScript( + absl::string_view script_name) const { + Delays arrival_delay_ms; + Delays corrected_arrival_delay_ms; + Delays playout_delay_ms; + Delays target_delay_ms; + CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms, + &playout_delay_ms, &target_delay_ms); + + // Maybe better to find the actually smallest timestamp, to surely avoid + // x-axis starting from negative. + const int64_t ref_time_ms = arrival_delay_ms.front().first; + + // Create an output file stream to Matlab script file. + std::ofstream output(std::string{script_name}); + + PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX, + kArrivalDelayY, output, ";"); + + // PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX, + // kCorrectedArrivalDelayY, output); + + PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY, + output, ";"); + + PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY, + output, ";"); + + output << "h=plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", " + << kTargetDelayX << ", " << kTargetDelayY << ", 'g.', " + << kPlayoutDelayX << ", " << kPlayoutDelayY << ");" << std::endl; + output << "set(h(1),'color',0.75*[1 1 1]);" << std::endl; + output << "set(h(2),'markersize',6);" << std::endl; + output << "set(h(3),'linew',1.5);" << std::endl; + output << "ax1=axis;" << std::endl; + output << "axis tight" << std::endl; + output << "ax2=axis;" << std::endl; + output << "axis([ax2(1:3) ax1(4)])" << std::endl; + output << "xlabel('time [s]');" << std::endl; + output << "ylabel('relative delay [ms]');" << std::endl; + if (!ssrcs_.empty()) { + auto ssrc_it = ssrcs_.cbegin(); + output << "title('SSRC: 0x" << std::hex << static_cast(*ssrc_it++); + while (ssrc_it != ssrcs_.end()) { + output << ", 0x" << std::hex << static_cast(*ssrc_it++); + } + output << std::dec; + auto pt_it = payload_types_.cbegin(); + output << "; Payload Types: " << *pt_it++; + while (pt_it != payload_types_.end()) { + output << ", " << *pt_it++; + } + output << "');" << std::endl; + } +} + +void NetEqDelayAnalyzer::CreatePythonScript( + absl::string_view script_name) const { + Delays arrival_delay_ms; + Delays corrected_arrival_delay_ms; + Delays playout_delay_ms; + Delays target_delay_ms; + CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms, + &playout_delay_ms, &target_delay_ms); + + // Maybe better to find the actually smallest timestamp, to surely avoid + // x-axis starting from negative. + const int64_t ref_time_ms = arrival_delay_ms.front().first; + + // Create an output file stream to the python script file. + std::ofstream output(std::string{script_name}); + + // Necessary includes + output << "import numpy as np" << std::endl; + output << "import matplotlib.pyplot as plt" << std::endl; + + PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX, + kArrivalDelayY, output); + + // PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX, + // kCorrectedArrivalDelayY, output); + + PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY, + output); + + PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY, + output); + + output << "if __name__ == '__main__':" << std::endl; + output << " h=plt.plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", " + << kTargetDelayX << ", " << kTargetDelayY << ", 'g.', " + << kPlayoutDelayX << ", " << kPlayoutDelayY << ")" << std::endl; + output << " plt.setp(h[0],'color',[.75, .75, .75])" << std::endl; + output << " plt.setp(h[1],'markersize',6)" << std::endl; + output << " plt.setp(h[2],'linewidth',1.5)" << std::endl; + output << " plt.axis('tight')" << std::endl; + output << " plt.xlabel('time [s]')" << std::endl; + output << " plt.ylabel('relative delay [ms]')" << std::endl; + if (!ssrcs_.empty()) { + auto ssrc_it = ssrcs_.cbegin(); + output << " plt.legend((\"arrival delay\", \"target delay\", \"playout " + "delay\"))" + << std::endl; + output << " plt.title('SSRC: 0x" << std::hex + << static_cast(*ssrc_it++); + while (ssrc_it != ssrcs_.end()) { + output << ", 0x" << std::hex << static_cast(*ssrc_it++); + } + output << std::dec; + auto pt_it = payload_types_.cbegin(); + output << "; Payload Types: " << *pt_it++; + while (pt_it != payload_types_.end()) { + output << ", " << *pt_it++; + } + output << "')" << std::endl; + } + output << " plt.show()" << std::endl; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h new file mode 100644 index 0000000000..ffcba5843f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_DELAY_ANALYZER_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_DELAY_ANALYZER_H_ + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" + +namespace webrtc { +namespace test { + +class NetEqDelayAnalyzer : public test::NetEqPostInsertPacket, + public test::NetEqGetAudioCallback { + public: + void AfterInsertPacket(const test::NetEqInput::PacketData& packet, + NetEq* neteq) override; + + void BeforeGetAudio(NetEq* neteq) override; + + void AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool muted, + NetEq* neteq) override; + + using Delays = std::vector>; + void CreateGraphs(Delays* arrival_delay_ms, + Delays* corrected_arrival_delay_ms, + Delays* playout_delay_ms, + Delays* target_delay_ms) const; + + // Creates a matlab script with file name script_name. When executed in + // Matlab, the script will generate graphs with the same timing information + // as provided by CreateGraphs. + void CreateMatlabScript(absl::string_view script_name) const; + + // Creates a python script with file name `script_name`. When executed in + // Python, the script will generate graphs with the same timing information + // as provided by CreateGraphs. + void CreatePythonScript(absl::string_view script_name) const; + + private: + struct TimingData { + explicit TimingData(int64_t at) : arrival_time_ms(at) {} + int64_t arrival_time_ms; + absl::optional decode_get_audio_count; + absl::optional sync_delay_ms; + absl::optional target_delay_ms; + absl::optional current_delay_ms; + }; + std::map data_; + std::vector get_audio_time_ms_; + size_t get_audio_count_ = 0; + size_t last_sync_buffer_ms_ = 0; + int last_sample_rate_hz_ = 0; + std::set ssrcs_; + std::set payload_types_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_DELAY_ANALYZER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_event_log_input.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_event_log_input.cc new file mode 100644 index 0000000000..0c1f27799a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_event_log_input.cc @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_event_log_input.h" + +#include +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/neteq/tools/rtc_event_log_source.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +NetEqEventLogInput* NetEqEventLogInput::CreateFromFile( + absl::string_view file_name, + absl::optional ssrc_filter) { + auto event_log_src = + RtcEventLogSource::CreateFromFile(file_name, ssrc_filter); + if (!event_log_src) { + return nullptr; + } + return new NetEqEventLogInput(std::move(event_log_src)); +} + +NetEqEventLogInput* NetEqEventLogInput::CreateFromString( + absl::string_view file_contents, + absl::optional ssrc_filter) { + auto event_log_src = + RtcEventLogSource::CreateFromString(file_contents, ssrc_filter); + if (!event_log_src) { + return nullptr; + } + return new NetEqEventLogInput(std::move(event_log_src)); +} + +absl::optional NetEqEventLogInput::NextOutputEventTime() const { + return next_output_event_ms_; +} + +void NetEqEventLogInput::AdvanceOutputEvent() { + next_output_event_ms_ = source_->NextAudioOutputEventMs(); + if (*next_output_event_ms_ == std::numeric_limits::max()) { + next_output_event_ms_ = absl::nullopt; + } +} + +PacketSource* NetEqEventLogInput::source() { + return source_.get(); +} + +NetEqEventLogInput::NetEqEventLogInput( + std::unique_ptr source) + : source_(std::move(source)) { + LoadNextPacket(); + AdvanceOutputEvent(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_event_log_input.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_event_log_input.h new file mode 100644 index 0000000000..c947ee1fc0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_event_log_input.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_EVENT_LOG_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_EVENT_LOG_INPUT_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/neteq/tools/neteq_packet_source_input.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { +namespace test { + +class RtcEventLogSource; + +// Implementation of NetEqPacketSourceInput to be used with an +// RtcEventLogSource. +class NetEqEventLogInput final : public NetEqPacketSourceInput { + public: + static NetEqEventLogInput* CreateFromFile( + absl::string_view file_name, + absl::optional ssrc_filter); + static NetEqEventLogInput* CreateFromString( + absl::string_view file_contents, + absl::optional ssrc_filter); + + absl::optional NextOutputEventTime() const override; + void AdvanceOutputEvent() override; + + protected: + PacketSource* source() override; + + private: + NetEqEventLogInput(std::unique_ptr source); + std::unique_ptr source_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_EVENT_LOG_INPUT_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_input.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_input.cc new file mode 100644 index 0000000000..de416348f1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_input.cc @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_input.h" + +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace test { + +NetEqInput::PacketData::PacketData() = default; +NetEqInput::PacketData::~PacketData() = default; + +std::string NetEqInput::PacketData::ToString() const { + rtc::StringBuilder ss; + ss << "{" + "time_ms: " + << static_cast(time_ms) + << ", " + "header: {" + "pt: " + << static_cast(header.payloadType) + << ", " + "sn: " + << header.sequenceNumber + << ", " + "ts: " + << header.timestamp + << ", " + "ssrc: " + << header.ssrc + << "}, " + "payload bytes: " + << payload.size() << "}"; + return ss.Release(); +} + +TimeLimitedNetEqInput::TimeLimitedNetEqInput(std::unique_ptr input, + int64_t duration_ms) + : input_(std::move(input)), + start_time_ms_(input_->NextEventTime()), + duration_ms_(duration_ms) {} + +TimeLimitedNetEqInput::~TimeLimitedNetEqInput() = default; + +absl::optional TimeLimitedNetEqInput::NextPacketTime() const { + return ended_ ? absl::nullopt : input_->NextPacketTime(); +} + +absl::optional TimeLimitedNetEqInput::NextOutputEventTime() const { + return ended_ ? absl::nullopt : input_->NextOutputEventTime(); +} + +std::unique_ptr TimeLimitedNetEqInput::PopPacket() { + if (ended_) { + return std::unique_ptr(); + } + auto packet = input_->PopPacket(); + MaybeSetEnded(); + return packet; +} + +void TimeLimitedNetEqInput::AdvanceOutputEvent() { + if (!ended_) { + input_->AdvanceOutputEvent(); + MaybeSetEnded(); + } +} + +bool TimeLimitedNetEqInput::ended() const { + return ended_ || input_->ended(); +} + +absl::optional TimeLimitedNetEqInput::NextHeader() const { + return ended_ ? absl::nullopt : input_->NextHeader(); +} + +void TimeLimitedNetEqInput::MaybeSetEnded() { + if (NextEventTime() && start_time_ms_ && + *NextEventTime() - *start_time_ms_ > duration_ms_) { + ended_ = true; + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_input.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_input.h new file mode 100644 index 0000000000..3a66264043 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_input.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_INPUT_H_ + +#include +#include +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "rtc_base/buffer.h" + +namespace webrtc { +namespace test { + +// Interface class for input to the NetEqTest class. +class NetEqInput { + public: + struct PacketData { + PacketData(); + ~PacketData(); + std::string ToString() const; + + RTPHeader header; + rtc::Buffer payload; + int64_t time_ms; + }; + + virtual ~NetEqInput() = default; + + // Returns at what time (in ms) NetEq::InsertPacket should be called next, or + // empty if the source is out of packets. + virtual absl::optional NextPacketTime() const = 0; + + // Returns at what time (in ms) NetEq::GetAudio should be called next, or + // empty if no more output events are available. + virtual absl::optional NextOutputEventTime() const = 0; + + // Returns the time (in ms) for the next event from either NextPacketTime() + // or NextOutputEventTime(), or empty if both are out of events. + absl::optional NextEventTime() const { + const auto a = NextPacketTime(); + const auto b = NextOutputEventTime(); + // Return the minimum of non-empty `a` and `b`, or empty if both are empty. + if (a) { + return b ? std::min(*a, *b) : a; + } + return b ? b : absl::nullopt; + } + + // Returns the next packet to be inserted into NetEq. The packet following the + // returned one is pre-fetched in the NetEqInput object, such that future + // calls to NextPacketTime() or NextHeader() will return information from that + // packet. + virtual std::unique_ptr PopPacket() = 0; + + // Move to the next output event. This will make NextOutputEventTime() return + // a new value (potentially the same if several output events share the same + // time). + virtual void AdvanceOutputEvent() = 0; + + // Returns true if the source has come to an end. An implementation must + // eventually return true from this method, or the test will end up in an + // infinite loop. + virtual bool ended() const = 0; + + // Returns the RTP header for the next packet, i.e., the packet that will be + // delivered next by PopPacket(). + virtual absl::optional NextHeader() const = 0; +}; + +// Wrapper class to impose a time limit on a NetEqInput object, typically +// another time limit than what the object itself provides. For example, an +// input taken from a file can be cut shorter by wrapping it in this class. +class TimeLimitedNetEqInput : public NetEqInput { + public: + TimeLimitedNetEqInput(std::unique_ptr input, int64_t duration_ms); + ~TimeLimitedNetEqInput() override; + absl::optional NextPacketTime() const override; + absl::optional NextOutputEventTime() const override; + std::unique_ptr PopPacket() override; + void AdvanceOutputEvent() override; + bool ended() const override; + absl::optional NextHeader() const override; + + private: + void MaybeSetEnded(); + + std::unique_ptr input_; + const absl::optional start_time_ms_; + const int64_t duration_ms_; + bool ended_ = false; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_INPUT_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc new file mode 100644 index 0000000000..55a5653238 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_packet_source_input.h" + +#include +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +NetEqPacketSourceInput::NetEqPacketSourceInput() : next_output_event_ms_(0) {} + +absl::optional NetEqPacketSourceInput::NextPacketTime() const { + return packet_ + ? absl::optional(static_cast(packet_->time_ms())) + : absl::nullopt; +} + +absl::optional NetEqPacketSourceInput::NextHeader() const { + return packet_ ? absl::optional(packet_->header()) : absl::nullopt; +} + +void NetEqPacketSourceInput::LoadNextPacket() { + packet_ = source()->NextPacket(); +} + +std::unique_ptr NetEqPacketSourceInput::PopPacket() { + if (!packet_) { + return std::unique_ptr(); + } + std::unique_ptr packet_data(new PacketData); + packet_data->header = packet_->header(); + if (packet_->payload_length_bytes() == 0 && + packet_->virtual_payload_length_bytes() > 0) { + // This is a header-only "dummy" packet. Set the payload to all zeros, with + // length according to the virtual length. + packet_data->payload.SetSize(packet_->virtual_payload_length_bytes()); + std::fill_n(packet_data->payload.data(), packet_data->payload.size(), 0); + } else { + packet_data->payload.SetData(packet_->payload(), + packet_->payload_length_bytes()); + } + packet_data->time_ms = packet_->time_ms(); + + LoadNextPacket(); + + return packet_data; +} + +NetEqRtpDumpInput::NetEqRtpDumpInput(absl::string_view file_name, + const RtpHeaderExtensionMap& hdr_ext_map, + absl::optional ssrc_filter) + : source_(RtpFileSource::Create(file_name, ssrc_filter)) { + for (const auto& ext_pair : hdr_ext_map) { + source_->RegisterRtpHeaderExtension(ext_pair.second, ext_pair.first); + } + LoadNextPacket(); +} + +absl::optional NetEqRtpDumpInput::NextOutputEventTime() const { + return next_output_event_ms_; +} + +void NetEqRtpDumpInput::AdvanceOutputEvent() { + if (next_output_event_ms_) { + *next_output_event_ms_ += kOutputPeriodMs; + } + if (!NextPacketTime()) { + next_output_event_ms_ = absl::nullopt; + } +} + +PacketSource* NetEqRtpDumpInput::source() { + return source_.get(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.h new file mode 100644 index 0000000000..407fa491b1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_packet_source_input.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PACKET_SOURCE_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PACKET_SOURCE_INPUT_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { +namespace test { + +class RtpFileSource; + +// An adapter class to dress up a PacketSource object as a NetEqInput. +class NetEqPacketSourceInput : public NetEqInput { + public: + using RtpHeaderExtensionMap = std::map; + + NetEqPacketSourceInput(); + absl::optional NextPacketTime() const override; + std::unique_ptr PopPacket() override; + absl::optional NextHeader() const override; + bool ended() const override { return !next_output_event_ms_; } + + protected: + virtual PacketSource* source() = 0; + void LoadNextPacket(); + + absl::optional next_output_event_ms_; + + private: + std::unique_ptr packet_; +}; + +// Implementation of NetEqPacketSourceInput to be used with an RtpFileSource. +class NetEqRtpDumpInput final : public NetEqPacketSourceInput { + public: + NetEqRtpDumpInput(absl::string_view file_name, + const RtpHeaderExtensionMap& hdr_ext_map, + absl::optional ssrc_filter); + + absl::optional NextOutputEventTime() const override; + void AdvanceOutputEvent() override; + + protected: + PacketSource* source() override; + + private: + static constexpr int64_t kOutputPeriodMs = 10; + + std::unique_ptr source_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PACKET_SOURCE_INPUT_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_performance_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_performance_test.cc new file mode 100644 index 0000000000..ccaa87b5e8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_performance_test.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_performance_test.h" + +#include "api/audio/audio_frame.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/neteq/neteq.h" +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/neteq/default_neteq_factory.h" +#include "modules/audio_coding/neteq/tools/audio_loop.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/clock.h" +#include "test/testsupport/file_utils.h" + +using webrtc::NetEq; +using webrtc::test::AudioLoop; +using webrtc::test::RtpGenerator; + +namespace webrtc { +namespace test { + +int64_t NetEqPerformanceTest::Run(int runtime_ms, + int lossrate, + double drift_factor) { + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + const int kSampRateHz = 32000; + const std::string kDecoderName = "pcm16-swb32"; + const int kPayloadType = 95; + + // Initialize NetEq instance. + NetEq::Config config; + config.sample_rate_hz = kSampRateHz; + webrtc::Clock* clock = webrtc::Clock::GetRealTimeClock(); + auto audio_decoder_factory = CreateBuiltinAudioDecoderFactory(); + auto neteq = + DefaultNetEqFactory().CreateNetEq(config, audio_decoder_factory, clock); + // Register decoder in `neteq`. + if (!neteq->RegisterPayloadType(kPayloadType, + SdpAudioFormat("l16", kSampRateHz, 1))) + return -1; + + // Set up AudioLoop object. + AudioLoop audio_loop; + const size_t kMaxLoopLengthSamples = kSampRateHz * 10; // 10 second loop. + const size_t kInputBlockSizeSamples = 60 * kSampRateHz / 1000; // 60 ms. + if (!audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)) + return -1; + + int32_t time_now_ms = 0; + + // Get first input packet. + RTPHeader rtp_header; + RtpGenerator rtp_gen(kSampRateHz / 1000); + // Start with positive drift first half of simulation. + rtp_gen.set_drift_factor(drift_factor); + bool drift_flipped = false; + int32_t packet_input_time_ms = + rtp_gen.GetRtpHeader(kPayloadType, kInputBlockSizeSamples, &rtp_header); + auto input_samples = audio_loop.GetNextBlock(); + if (input_samples.empty()) + exit(1); + uint8_t input_payload[kInputBlockSizeSamples * sizeof(int16_t)]; + size_t payload_len = WebRtcPcm16b_Encode(input_samples.data(), + input_samples.size(), input_payload); + RTC_CHECK_EQ(sizeof(input_payload), payload_len); + + // Main loop. + int64_t start_time_ms = clock->TimeInMilliseconds(); + AudioFrame out_frame; + while (time_now_ms < runtime_ms) { + while (packet_input_time_ms <= time_now_ms) { + // Drop every N packets, where N = FLAG_lossrate. + bool lost = false; + if (lossrate > 0) { + lost = ((rtp_header.sequenceNumber - 1) % lossrate) == 0; + } + if (!lost) { + // Insert packet. + int error = neteq->InsertPacket(rtp_header, input_payload); + if (error != NetEq::kOK) + return -1; + } + + // Get next packet. + packet_input_time_ms = rtp_gen.GetRtpHeader( + kPayloadType, kInputBlockSizeSamples, &rtp_header); + input_samples = audio_loop.GetNextBlock(); + if (input_samples.empty()) + return -1; + payload_len = WebRtcPcm16b_Encode(input_samples.data(), + input_samples.size(), input_payload); + RTC_DCHECK_EQ(payload_len, kInputBlockSizeSamples * sizeof(int16_t)); + } + + // Get output audio, but don't do anything with it. + bool muted; + int error = neteq->GetAudio(&out_frame, &muted); + RTC_CHECK(!muted); + if (error != NetEq::kOK) + return -1; + + RTC_DCHECK_EQ(out_frame.samples_per_channel_, (kSampRateHz * 10) / 1000); + + static const int kOutputBlockSizeMs = 10; + time_now_ms += kOutputBlockSizeMs; + if (time_now_ms >= runtime_ms / 2 && !drift_flipped) { + // Apply negative drift second half of simulation. + rtp_gen.set_drift_factor(-drift_factor); + drift_flipped = true; + } + } + int64_t end_time_ms = clock->TimeInMilliseconds(); + return end_time_ms - start_time_ms; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_performance_test.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_performance_test.h new file mode 100644 index 0000000000..b5b4d91577 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_performance_test.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PERFORMANCE_TEST_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PERFORMANCE_TEST_H_ + +#include + +namespace webrtc { +namespace test { + +class NetEqPerformanceTest { + public: + // Runs a performance test with parameters as follows: + // `runtime_ms`: the simulation time, i.e., the duration of the audio data. + // `lossrate`: drop one out of `lossrate` packets, e.g., one out of 10. + // `drift_factor`: clock drift in [0, 1]. + // Returns the runtime in ms. + static int64_t Run(int runtime_ms, int lossrate, double drift_factor); +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_PERFORMANCE_TEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_quality_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_quality_test.cc new file mode 100644 index 0000000000..18e6091f93 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_quality_test.cc @@ -0,0 +1,482 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" + +#include + +#include + +#include "absl/flags/flag.h" +#include "absl/strings/string_view.h" +#include "modules/audio_coding/neteq/default_neteq_factory.h" +#include "modules/audio_coding/neteq/tools/neteq_quality_test.h" +#include "modules/audio_coding/neteq/tools/output_audio_file.h" +#include "modules/audio_coding/neteq/tools/output_wav_file.h" +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "rtc_base/checks.h" +#include "rtc_base/string_encode.h" +#include "system_wrappers/include/clock.h" +#include "test/testsupport/file_utils.h" + +ABSL_FLAG(std::string, + in_filename, + "audio_coding/speech_mono_16kHz.pcm", + "Path of the input file (relative to the resources/ directory) for " + "input audio (specify sample rate with --input_sample_rate, " + "and channels with --channels)."); + +ABSL_FLAG(int, input_sample_rate, 16000, "Sample rate of input file in Hz."); + +ABSL_FLAG(int, channels, 1, "Number of channels in input audio."); + +ABSL_FLAG(std::string, + out_filename, + "neteq_quality_test_out.pcm", + "Name of output audio file, which will be saved in " + + ::webrtc::test::OutputPath()); + +ABSL_FLAG( + int, + runtime_ms, + 10000, + "Simulated runtime (milliseconds). -1 will consume the complete file."); + +ABSL_FLAG(int, packet_loss_rate, 10, "Percentile of packet loss."); + +ABSL_FLAG(int, + random_loss_mode, + ::webrtc::test::kUniformLoss, + "Random loss mode: 0--no loss, 1--uniform loss, 2--Gilbert Elliot " + "loss, 3--fixed loss."); + +ABSL_FLAG(int, + burst_length, + 30, + "Burst length in milliseconds, only valid for Gilbert Elliot loss."); + +ABSL_FLAG(float, drift_factor, 0.0, "Time drift factor."); + +ABSL_FLAG(int, + preload_packets, + 1, + "Preload the buffer with this many packets."); + +ABSL_FLAG(std::string, + loss_events, + "", + "List of loss events time and duration separated by comma: " + " , " + ", ..."); + +namespace webrtc { +namespace test { + +namespace { + +std::unique_ptr CreateNetEq( + const NetEq::Config& config, + Clock* clock, + const rtc::scoped_refptr& decoder_factory) { + return DefaultNetEqFactory().CreateNetEq(config, decoder_factory, clock); +} + +const std::string& GetInFilenamePath(absl::string_view file_name) { + std::vector name_parts = rtc::split(file_name, '.'); + RTC_CHECK_EQ(name_parts.size(), 2); + static const std::string path = + ::webrtc::test::ResourcePath(name_parts[0], name_parts[1]); + return path; +} + +const std::string& GetOutFilenamePath(absl::string_view file_name) { + static const std::string path = + ::webrtc::test::OutputPath() + std::string(file_name); + return path; +} + +} // namespace + +const uint8_t kPayloadType = 95; +const int kOutputSizeMs = 10; +const int kInitSeed = 0x12345678; +const int kPacketLossTimeUnitMs = 10; + +// Common validator for file names. +static bool ValidateFilename(absl::string_view value, bool is_output) { + if (!is_output) { + RTC_CHECK_NE(value.substr(value.find_last_of('.') + 1), "wav") + << "WAV file input is not supported"; + } + FILE* fid = is_output ? fopen(std::string(value).c_str(), "wb") + : fopen(std::string(value).c_str(), "rb"); + if (fid == nullptr) + return false; + fclose(fid); + return true; +} + +// ProbTrans00Solver() is to calculate the transition probability from no-loss +// state to itself in a modified Gilbert Elliot packet loss model. The result is +// to achieve the target packet loss rate `loss_rate`, when a packet is not +// lost only if all `units` drawings within the duration of the packet result in +// no-loss. +static double ProbTrans00Solver(int units, + double loss_rate, + double prob_trans_10) { + if (units == 1) + return prob_trans_10 / (1.0f - loss_rate) - prob_trans_10; + // 0 == prob_trans_00 ^ (units - 1) + (1 - loss_rate) / prob_trans_10 * + // prob_trans_00 - (1 - loss_rate) * (1 + 1 / prob_trans_10). + // There is a unique solution between 0.0 and 1.0, due to the monotonicity and + // an opposite sign at 0.0 and 1.0. + // For simplicity, we reformulate the equation as + // f(x) = x ^ (units - 1) + a x + b. + // Its derivative is + // f'(x) = (units - 1) x ^ (units - 2) + a. + // The derivative is strictly greater than 0 when x is between 0 and 1. + // We use Newton's method to solve the equation, iteration is + // x(k+1) = x(k) - f(x) / f'(x); + const double kPrecision = 0.001f; + const int kIterations = 100; + const double a = (1.0f - loss_rate) / prob_trans_10; + const double b = (loss_rate - 1.0f) * (1.0f + 1.0f / prob_trans_10); + double x = 0.0; // Starting point; + double f = b; + double f_p; + int iter = 0; + while ((f >= kPrecision || f <= -kPrecision) && iter < kIterations) { + f_p = (units - 1.0f) * std::pow(x, units - 2) + a; + x -= f / f_p; + if (x > 1.0f) { + x = 1.0f; + } else if (x < 0.0f) { + x = 0.0f; + } + f = std::pow(x, units - 1) + a * x + b; + iter++; + } + return x; +} + +NetEqQualityTest::NetEqQualityTest( + int block_duration_ms, + int in_sampling_khz, + int out_sampling_khz, + const SdpAudioFormat& format, + const rtc::scoped_refptr& decoder_factory) + : audio_format_(format), + channels_(absl::GetFlag(FLAGS_channels)), + decoded_time_ms_(0), + decodable_time_ms_(0), + drift_factor_(absl::GetFlag(FLAGS_drift_factor)), + packet_loss_rate_(absl::GetFlag(FLAGS_packet_loss_rate)), + block_duration_ms_(block_duration_ms), + in_sampling_khz_(in_sampling_khz), + out_sampling_khz_(out_sampling_khz), + in_size_samples_( + static_cast(in_sampling_khz_ * block_duration_ms_)), + payload_size_bytes_(0), + max_payload_bytes_(0), + in_file_(new ResampleInputAudioFile( + GetInFilenamePath(absl::GetFlag(FLAGS_in_filename)), + absl::GetFlag(FLAGS_input_sample_rate), + in_sampling_khz * 1000, + absl::GetFlag(FLAGS_runtime_ms) > 0)), + rtp_generator_( + new RtpGenerator(in_sampling_khz_, 0, 0, decodable_time_ms_)), + total_payload_size_bytes_(0) { + // Flag validation + RTC_CHECK(ValidateFilename( + GetInFilenamePath(absl::GetFlag(FLAGS_in_filename)), false)) + << "Invalid input filename."; + + RTC_CHECK(absl::GetFlag(FLAGS_input_sample_rate) == 8000 || + absl::GetFlag(FLAGS_input_sample_rate) == 16000 || + absl::GetFlag(FLAGS_input_sample_rate) == 32000 || + absl::GetFlag(FLAGS_input_sample_rate) == 48000) + << "Invalid sample rate should be 8000, 16000, 32000 or 48000 Hz."; + + RTC_CHECK_EQ(absl::GetFlag(FLAGS_channels), 1) + << "Invalid number of channels, current support only 1."; + + RTC_CHECK(ValidateFilename( + GetOutFilenamePath(absl::GetFlag(FLAGS_out_filename)), true)) + << "Invalid output filename."; + + RTC_CHECK(absl::GetFlag(FLAGS_packet_loss_rate) >= 0 && + absl::GetFlag(FLAGS_packet_loss_rate) <= 100) + << "Invalid packet loss percentile, should be between 0 and 100."; + + RTC_CHECK(absl::GetFlag(FLAGS_random_loss_mode) >= 0 && + absl::GetFlag(FLAGS_random_loss_mode) < kLastLossMode) + << "Invalid random packet loss mode, should be between 0 and " + << kLastLossMode - 1 << "."; + + RTC_CHECK_GE(absl::GetFlag(FLAGS_burst_length), kPacketLossTimeUnitMs) + << "Invalid burst length, should be greater than or equal to " + << kPacketLossTimeUnitMs << " ms."; + + RTC_CHECK_GT(absl::GetFlag(FLAGS_drift_factor), -0.1) + << "Invalid drift factor, should be greater than -0.1."; + + RTC_CHECK_GE(absl::GetFlag(FLAGS_preload_packets), 0) + << "Invalid number of packets to preload; must be non-negative."; + + const std::string out_filename = + GetOutFilenamePath(absl::GetFlag(FLAGS_out_filename)); + const std::string log_filename = out_filename + ".log"; + log_file_.open(log_filename.c_str(), std::ofstream::out); + RTC_CHECK(log_file_.is_open()); + + if (out_filename.size() >= 4 && + out_filename.substr(out_filename.size() - 4) == ".wav") { + // Open a wav file. + output_.reset( + new webrtc::test::OutputWavFile(out_filename, 1000 * out_sampling_khz)); + } else { + // Open a pcm file. + output_.reset(new webrtc::test::OutputAudioFile(out_filename)); + } + + NetEq::Config config; + config.sample_rate_hz = out_sampling_khz_ * 1000; + neteq_ = CreateNetEq(config, Clock::GetRealTimeClock(), decoder_factory); + max_payload_bytes_ = in_size_samples_ * channels_ * sizeof(int16_t); + in_data_.reset(new int16_t[in_size_samples_ * channels_]); +} + +NetEqQualityTest::~NetEqQualityTest() { + log_file_.close(); +} + +bool NoLoss::Lost(int now_ms) { + return false; +} + +UniformLoss::UniformLoss(double loss_rate) : loss_rate_(loss_rate) {} + +bool UniformLoss::Lost(int now_ms) { + int drop_this = rand(); + return (drop_this < loss_rate_ * RAND_MAX); +} + +GilbertElliotLoss::GilbertElliotLoss(double prob_trans_11, double prob_trans_01) + : prob_trans_11_(prob_trans_11), + prob_trans_01_(prob_trans_01), + lost_last_(false), + uniform_loss_model_(new UniformLoss(0)) {} + +GilbertElliotLoss::~GilbertElliotLoss() {} + +bool GilbertElliotLoss::Lost(int now_ms) { + // Simulate bursty channel (Gilbert model). + // (1st order) Markov chain model with memory of the previous/last + // packet state (lost or received). + if (lost_last_) { + // Previous packet was not received. + uniform_loss_model_->set_loss_rate(prob_trans_11_); + return lost_last_ = uniform_loss_model_->Lost(now_ms); + } else { + uniform_loss_model_->set_loss_rate(prob_trans_01_); + return lost_last_ = uniform_loss_model_->Lost(now_ms); + } +} + +FixedLossModel::FixedLossModel( + std::set loss_events) + : loss_events_(loss_events) { + loss_events_it_ = loss_events_.begin(); +} + +FixedLossModel::~FixedLossModel() {} + +bool FixedLossModel::Lost(int now_ms) { + if (loss_events_it_ != loss_events_.end() && + now_ms > loss_events_it_->start_ms) { + if (now_ms <= loss_events_it_->start_ms + loss_events_it_->duration_ms) { + return true; + } else { + ++loss_events_it_; + return false; + } + } + return false; +} + +void NetEqQualityTest::SetUp() { + ASSERT_TRUE(neteq_->RegisterPayloadType(kPayloadType, audio_format_)); + rtp_generator_->set_drift_factor(drift_factor_); + + int units = block_duration_ms_ / kPacketLossTimeUnitMs; + switch (absl::GetFlag(FLAGS_random_loss_mode)) { + case kUniformLoss: { + // `unit_loss_rate` is the packet loss rate for each unit time interval + // (kPacketLossTimeUnitMs). Since a packet loss event is generated if any + // of |block_duration_ms_ / kPacketLossTimeUnitMs| unit time intervals of + // a full packet duration is drawn with a loss, `unit_loss_rate` fulfills + // (1 - unit_loss_rate) ^ (block_duration_ms_ / kPacketLossTimeUnitMs) == + // 1 - packet_loss_rate. + double unit_loss_rate = + (1.0 - std::pow(1.0 - 0.01 * packet_loss_rate_, 1.0 / units)); + loss_model_.reset(new UniformLoss(unit_loss_rate)); + break; + } + case kGilbertElliotLoss: { + // `FLAGS_burst_length` should be integer times of kPacketLossTimeUnitMs. + ASSERT_EQ(0, absl::GetFlag(FLAGS_burst_length) % kPacketLossTimeUnitMs); + + // We do not allow 100 percent packet loss in Gilbert Elliot model, which + // makes no sense. + ASSERT_GT(100, packet_loss_rate_); + + // To guarantee the overall packet loss rate, transition probabilities + // need to satisfy: + // pi_0 * (1 - prob_trans_01_) ^ units + + // pi_1 * prob_trans_10_ ^ (units - 1) == 1 - loss_rate + // pi_0 = prob_trans_10 / (prob_trans_10 + prob_trans_01_) + // is the stationary state probability of no-loss + // pi_1 = prob_trans_01_ / (prob_trans_10 + prob_trans_01_) + // is the stationary state probability of loss + // After a derivation prob_trans_00 should satisfy: + // prob_trans_00 ^ (units - 1) = (loss_rate - 1) / prob_trans_10 * + // prob_trans_00 + (1 - loss_rate) * (1 + 1 / prob_trans_10). + double loss_rate = 0.01f * packet_loss_rate_; + double prob_trans_10 = + 1.0f * kPacketLossTimeUnitMs / absl::GetFlag(FLAGS_burst_length); + double prob_trans_00 = ProbTrans00Solver(units, loss_rate, prob_trans_10); + loss_model_.reset( + new GilbertElliotLoss(1.0f - prob_trans_10, 1.0f - prob_trans_00)); + break; + } + case kFixedLoss: { + std::istringstream loss_events_stream(absl::GetFlag(FLAGS_loss_events)); + std::string loss_event_string; + std::set loss_events; + while (std::getline(loss_events_stream, loss_event_string, ',')) { + std::vector loss_event_params; + std::istringstream loss_event_params_stream(loss_event_string); + std::copy(std::istream_iterator(loss_event_params_stream), + std::istream_iterator(), + std::back_inserter(loss_event_params)); + RTC_CHECK_EQ(loss_event_params.size(), 2); + auto result = loss_events.insert( + FixedLossEvent(loss_event_params[0], loss_event_params[1])); + RTC_CHECK(result.second); + } + RTC_CHECK_GT(loss_events.size(), 0); + loss_model_.reset(new FixedLossModel(loss_events)); + break; + } + default: { + loss_model_.reset(new NoLoss); + break; + } + } + + // Make sure that the packet loss profile is same for all derived tests. + srand(kInitSeed); +} + +std::ofstream& NetEqQualityTest::Log() { + return log_file_; +} + +bool NetEqQualityTest::PacketLost() { + int cycles = block_duration_ms_ / kPacketLossTimeUnitMs; + + // The loop is to make sure that codecs with different block lengths share the + // same packet loss profile. + bool lost = false; + for (int idx = 0; idx < cycles; idx++) { + if (loss_model_->Lost(decoded_time_ms_)) { + // The packet will be lost if any of the drawings indicates a loss, but + // the loop has to go on to make sure that codecs with different block + // lengths keep the same pace. + lost = true; + } + } + return lost; +} + +int NetEqQualityTest::Transmit() { + int packet_input_time_ms = rtp_generator_->GetRtpHeader( + kPayloadType, in_size_samples_, &rtp_header_); + Log() << "Packet of size " << payload_size_bytes_ << " bytes, for frame at " + << packet_input_time_ms << " ms "; + if (payload_size_bytes_ > 0) { + if (!PacketLost()) { + int ret = neteq_->InsertPacket( + rtp_header_, + rtc::ArrayView(payload_.data(), payload_size_bytes_)); + if (ret != NetEq::kOK) + return -1; + Log() << "was sent."; + } else { + Log() << "was lost."; + } + } + Log() << std::endl; + return packet_input_time_ms; +} + +int NetEqQualityTest::DecodeBlock() { + bool muted; + int ret = neteq_->GetAudio(&out_frame_, &muted); + RTC_CHECK(!muted); + + if (ret != NetEq::kOK) { + return -1; + } else { + RTC_DCHECK_EQ(out_frame_.num_channels_, channels_); + RTC_DCHECK_EQ(out_frame_.samples_per_channel_, + static_cast(kOutputSizeMs * out_sampling_khz_)); + RTC_CHECK(output_->WriteArray( + out_frame_.data(), + out_frame_.samples_per_channel_ * out_frame_.num_channels_)); + return static_cast(out_frame_.samples_per_channel_); + } +} + +void NetEqQualityTest::Simulate() { + int audio_size_samples; + bool end_of_input = false; + int runtime_ms = absl::GetFlag(FLAGS_runtime_ms) >= 0 + ? absl::GetFlag(FLAGS_runtime_ms) + : INT_MAX; + + while (!end_of_input && decoded_time_ms_ < runtime_ms) { + // Preload the buffer if needed. + while (decodable_time_ms_ - + absl::GetFlag(FLAGS_preload_packets) * block_duration_ms_ < + decoded_time_ms_) { + if (!in_file_->Read(in_size_samples_ * channels_, &in_data_[0])) { + end_of_input = true; + ASSERT_TRUE(end_of_input && absl::GetFlag(FLAGS_runtime_ms) < 0); + break; + } + payload_.Clear(); + payload_size_bytes_ = EncodeBlock(&in_data_[0], in_size_samples_, + &payload_, max_payload_bytes_); + total_payload_size_bytes_ += payload_size_bytes_; + decodable_time_ms_ = Transmit() + block_duration_ms_; + } + audio_size_samples = DecodeBlock(); + if (audio_size_samples > 0) { + decoded_time_ms_ += audio_size_samples / out_sampling_khz_; + } + } + Log() << "Average bit rate was " + << 8.0f * total_payload_size_bytes_ / absl::GetFlag(FLAGS_runtime_ms) + << " kbps" << std::endl; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_quality_test.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_quality_test.h new file mode 100644 index 0000000000..edcb117748 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_quality_test.h @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_QUALITY_TEST_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_QUALITY_TEST_H_ + +#include +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/neteq/neteq.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/rtp_generator.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +enum LossModes { + kNoLoss, + kUniformLoss, + kGilbertElliotLoss, + kFixedLoss, + kLastLossMode +}; + +class LossModel { + public: + virtual ~LossModel() {} + virtual bool Lost(int now_ms) = 0; +}; + +class NoLoss : public LossModel { + public: + bool Lost(int now_ms) override; +}; + +class UniformLoss : public LossModel { + public: + UniformLoss(double loss_rate); + bool Lost(int now_ms) override; + void set_loss_rate(double loss_rate) { loss_rate_ = loss_rate; } + + private: + double loss_rate_; +}; + +class GilbertElliotLoss : public LossModel { + public: + GilbertElliotLoss(double prob_trans_11, double prob_trans_01); + ~GilbertElliotLoss() override; + bool Lost(int now_ms) override; + + private: + // Prob. of losing current packet, when previous packet is lost. + double prob_trans_11_; + // Prob. of losing current packet, when previous packet is not lost. + double prob_trans_01_; + bool lost_last_; + std::unique_ptr uniform_loss_model_; +}; + +struct FixedLossEvent { + int start_ms; + int duration_ms; + FixedLossEvent(int start_ms, int duration_ms) + : start_ms(start_ms), duration_ms(duration_ms) {} +}; + +struct FixedLossEventCmp { + bool operator()(const FixedLossEvent& l_event, + const FixedLossEvent& r_event) const { + return l_event.start_ms < r_event.start_ms; + } +}; + +class FixedLossModel : public LossModel { + public: + FixedLossModel(std::set loss_events); + ~FixedLossModel() override; + bool Lost(int now_ms) override; + + private: + std::set loss_events_; + std::set::iterator loss_events_it_; +}; + +class NetEqQualityTest : public ::testing::Test { + protected: + NetEqQualityTest( + int block_duration_ms, + int in_sampling_khz, + int out_sampling_khz, + const SdpAudioFormat& format, + const rtc::scoped_refptr& decoder_factory = + webrtc::CreateBuiltinAudioDecoderFactory()); + ~NetEqQualityTest() override; + + void SetUp() override; + + // EncodeBlock(...) does the following: + // 1. encodes a block of audio, saved in `in_data` and has a length of + // `block_size_samples` (samples per channel), + // 2. save the bit stream to `payload` of `max_bytes` bytes in size, + // 3. returns the length of the payload (in bytes), + virtual int EncodeBlock(int16_t* in_data, + size_t block_size_samples, + rtc::Buffer* payload, + size_t max_bytes) = 0; + + // PacketLost(...) determines weather a packet sent at an indicated time gets + // lost or not. + bool PacketLost(); + + // DecodeBlock() decodes a block of audio using the payload stored in + // `payload_` with the length of `payload_size_bytes_` (bytes). The decoded + // audio is to be stored in `out_data_`. + int DecodeBlock(); + + // Transmit() uses `rtp_generator_` to generate a packet and passes it to + // `neteq_`. + int Transmit(); + + // Runs encoding / transmitting / decoding. + void Simulate(); + + // Write to log file. Usage Log() << ... + std::ofstream& Log(); + + SdpAudioFormat audio_format_; + const size_t channels_; + + private: + int decoded_time_ms_; + int decodable_time_ms_; + double drift_factor_; + int packet_loss_rate_; + const int block_duration_ms_; + const int in_sampling_khz_; + const int out_sampling_khz_; + + // Number of samples per channel in a frame. + const size_t in_size_samples_; + + size_t payload_size_bytes_; + size_t max_payload_bytes_; + + std::unique_ptr in_file_; + std::unique_ptr output_; + std::ofstream log_file_; + + std::unique_ptr rtp_generator_; + std::unique_ptr neteq_; + std::unique_ptr loss_model_; + + std::unique_ptr in_data_; + rtc::Buffer payload_; + AudioFrame out_frame_; + RTPHeader rtp_header_; + + size_t total_payload_size_bytes_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_QUALITY_TEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.cc new file mode 100644 index 0000000000..ffd114ae5b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.cc @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_replacement_input.h" + +#include "modules/audio_coding/neteq/tools/fake_decode_from_file.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +NetEqReplacementInput::NetEqReplacementInput( + std::unique_ptr source, + uint8_t replacement_payload_type, + const std::set& comfort_noise_types, + const std::set& forbidden_types) + : source_(std::move(source)), + replacement_payload_type_(replacement_payload_type), + comfort_noise_types_(comfort_noise_types), + forbidden_types_(forbidden_types) { + RTC_CHECK(source_); + packet_ = source_->PopPacket(); + ReplacePacket(); +} + +absl::optional NetEqReplacementInput::NextPacketTime() const { + return packet_ + ? absl::optional(static_cast(packet_->time_ms)) + : absl::nullopt; +} + +absl::optional NetEqReplacementInput::NextOutputEventTime() const { + return source_->NextOutputEventTime(); +} + +std::unique_ptr NetEqReplacementInput::PopPacket() { + std::unique_ptr to_return = std::move(packet_); + while (true) { + packet_ = source_->PopPacket(); + if (!packet_) + break; + if (packet_->payload.size() > packet_->header.paddingLength) { + // Not padding only. Good to go. Skip this packet otherwise. + break; + } + } + ReplacePacket(); + return to_return; +} + +void NetEqReplacementInput::AdvanceOutputEvent() { + source_->AdvanceOutputEvent(); +} + +bool NetEqReplacementInput::ended() const { + return source_->ended(); +} + +absl::optional NetEqReplacementInput::NextHeader() const { + return source_->NextHeader(); +} + +void NetEqReplacementInput::ReplacePacket() { + if (!source_->NextPacketTime()) { + // End of input. Cannot do proper replacement on the very last packet, so we + // delete it instead. + packet_.reset(); + return; + } + + RTC_DCHECK(packet_); + + RTC_CHECK_EQ(forbidden_types_.count(packet_->header.payloadType), 0) + << "Payload type " << static_cast(packet_->header.payloadType) + << " is forbidden."; + + // Check if this packet is comfort noise. + if (comfort_noise_types_.count(packet_->header.payloadType) != 0) { + // If CNG, simply insert a zero-energy one-byte payload. + uint8_t cng_payload[1] = {127}; // Max attenuation of CNG. + packet_->payload.SetData(cng_payload); + return; + } + + absl::optional next_hdr = source_->NextHeader(); + RTC_DCHECK(next_hdr); + uint8_t payload[12]; + RTC_DCHECK_LE(last_frame_size_timestamps_, 120 * 48); + uint32_t input_frame_size_timestamps = last_frame_size_timestamps_; + const uint32_t timestamp_diff = + next_hdr->timestamp - packet_->header.timestamp; + if (next_hdr->sequenceNumber == packet_->header.sequenceNumber + 1 && + timestamp_diff <= 120 * 48) { + // Packets are in order and the timestamp diff is less than 5760 samples. + // Accept the timestamp diff as a valid frame size. + input_frame_size_timestamps = timestamp_diff; + last_frame_size_timestamps_ = input_frame_size_timestamps; + } + RTC_DCHECK_LE(input_frame_size_timestamps, 120 * 48); + FakeDecodeFromFile::PrepareEncoded(packet_->header.timestamp, + input_frame_size_timestamps, + packet_->payload.size(), payload); + packet_->payload.SetData(payload); + packet_->header.payloadType = replacement_payload_type_; + return; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h new file mode 100644 index 0000000000..9ce9b9dc63 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_REPLACEMENT_INPUT_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_REPLACEMENT_INPUT_H_ + +#include +#include + +#include "modules/audio_coding/neteq/tools/neteq_input.h" + +namespace webrtc { +namespace test { + +// This class converts the packets from a NetEqInput to fake encodings to be +// decoded by a FakeDecodeFromFile decoder. +class NetEqReplacementInput : public NetEqInput { + public: + NetEqReplacementInput(std::unique_ptr source, + uint8_t replacement_payload_type, + const std::set& comfort_noise_types, + const std::set& forbidden_types); + + absl::optional NextPacketTime() const override; + absl::optional NextOutputEventTime() const override; + std::unique_ptr PopPacket() override; + void AdvanceOutputEvent() override; + bool ended() const override; + absl::optional NextHeader() const override; + + private: + void ReplacePacket(); + + std::unique_ptr source_; + const uint8_t replacement_payload_type_; + const std::set comfort_noise_types_; + const std::set forbidden_types_; + std::unique_ptr packet_; // The next packet to deliver. + uint32_t last_frame_size_timestamps_ = 960; // Initial guess: 20 ms @ 48 kHz. +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_REPLACEMENT_INPUT_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc new file mode 100644 index 0000000000..b274069bd4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_rtpplay.cc @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" +#include "modules/audio_coding/neteq/tools/neteq_test_factory.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/field_trial.h" +#include "test/field_trial.h" + +using TestConfig = webrtc::test::NetEqTestFactory::Config; + +ABSL_FLAG(bool, + codec_map, + false, + "Prints the mapping between RTP payload type and " + "codec"); +ABSL_FLAG(std::string, + force_fieldtrials, + "", + "Field trials control experimental feature code which can be forced. " + "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/" + " will assign the group Enable to field trial WebRTC-FooFeature."); +ABSL_FLAG(int, pcmu, TestConfig::default_pcmu(), "RTP payload type for PCM-u"); +ABSL_FLAG(int, pcma, TestConfig::default_pcma(), "RTP payload type for PCM-a"); +ABSL_FLAG(int, ilbc, TestConfig::default_ilbc(), "RTP payload type for iLBC"); +ABSL_FLAG(int, isac, TestConfig::default_isac(), "RTP payload type for iSAC"); +ABSL_FLAG(int, + isac_swb, + TestConfig::default_isac_swb(), + "RTP payload type for iSAC-swb (32 kHz)"); +ABSL_FLAG(int, opus, TestConfig::default_opus(), "RTP payload type for Opus"); +ABSL_FLAG(int, + pcm16b, + TestConfig::default_pcm16b(), + "RTP payload type for PCM16b-nb (8 kHz)"); +ABSL_FLAG(int, + pcm16b_wb, + TestConfig::default_pcm16b_wb(), + "RTP payload type for PCM16b-wb (16 kHz)"); +ABSL_FLAG(int, + pcm16b_swb32, + TestConfig::default_pcm16b_swb32(), + "RTP payload type for PCM16b-swb32 (32 kHz)"); +ABSL_FLAG(int, + pcm16b_swb48, + TestConfig::default_pcm16b_swb48(), + "RTP payload type for PCM16b-swb48 (48 kHz)"); +ABSL_FLAG(int, g722, TestConfig::default_g722(), "RTP payload type for G.722"); +ABSL_FLAG(int, + avt, + TestConfig::default_avt(), + "RTP payload type for AVT/DTMF (8 kHz)"); +ABSL_FLAG(int, + avt_16, + TestConfig::default_avt_16(), + "RTP payload type for AVT/DTMF (16 kHz)"); +ABSL_FLAG(int, + avt_32, + TestConfig::default_avt_32(), + "RTP payload type for AVT/DTMF (32 kHz)"); +ABSL_FLAG(int, + avt_48, + TestConfig::default_avt_48(), + "RTP payload type for AVT/DTMF (48 kHz)"); +ABSL_FLAG(int, + red, + TestConfig::default_red(), + "RTP payload type for redundant audio (RED)"); +ABSL_FLAG(int, + cn_nb, + TestConfig::default_cn_nb(), + "RTP payload type for comfort noise (8 kHz)"); +ABSL_FLAG(int, + cn_wb, + TestConfig::default_cn_wb(), + "RTP payload type for comfort noise (16 kHz)"); +ABSL_FLAG(int, + cn_swb32, + TestConfig::default_cn_swb32(), + "RTP payload type for comfort noise (32 kHz)"); +ABSL_FLAG(int, + cn_swb48, + TestConfig::default_cn_swb48(), + "RTP payload type for comfort noise (48 kHz)"); +ABSL_FLAG(std::string, + replacement_audio_file, + "", + "A PCM file that will be used to populate dummy" + " RTP packets"); +ABSL_FLAG(std::string, + ssrc, + "", + "Only use packets with this SSRC (decimal or hex, the latter " + "starting with 0x)"); +ABSL_FLAG(int, + audio_level, + TestConfig::default_audio_level(), + "Extension ID for audio level (RFC 6464)"); +ABSL_FLAG(int, + abs_send_time, + TestConfig::default_abs_send_time(), + "Extension ID for absolute sender time"); +ABSL_FLAG(int, + transport_seq_no, + TestConfig::default_transport_seq_no(), + "Extension ID for transport sequence number"); +ABSL_FLAG(int, + video_content_type, + TestConfig::default_video_content_type(), + "Extension ID for video content type"); +ABSL_FLAG(int, + video_timing, + TestConfig::default_video_timing(), + "Extension ID for video timing"); +ABSL_FLAG(std::string, + output_files_base_name, + "", + "Custom path used as prefix for the output files - i.e., " + "matlab plot, python plot, text log."); +ABSL_FLAG(bool, + matlabplot, + false, + "Generates a matlab script for plotting the delay profile"); +ABSL_FLAG(bool, + pythonplot, + false, + "Generates a python script for plotting the delay profile"); +ABSL_FLAG(bool, + textlog, + false, + "Generates a text log describing the simulation on a " + "step-by-step basis."); +ABSL_FLAG(bool, concealment_events, false, "Prints concealment events"); +ABSL_FLAG(int, + max_nr_packets_in_buffer, + TestConfig::default_max_nr_packets_in_buffer(), + "Maximum allowed number of packets in the buffer"); +ABSL_FLAG(bool, + enable_fast_accelerate, + false, + "Enables jitter buffer fast accelerate"); + +namespace { + +// Parses the input string for a valid SSRC (at the start of the string). If a +// valid SSRC is found, it is written to the output variable `ssrc`, and true is +// returned. Otherwise, false is returned. +bool ParseSsrc(absl::string_view str, uint32_t* ssrc) { + if (str.empty()) + return true; + int base = 10; + // Look for "0x" or "0X" at the start and change base to 16 if found. + if ((str.compare(0, 2, "0x") == 0) || (str.compare(0, 2, "0X") == 0)) + base = 16; + errno = 0; + char* end_ptr; + std::string str_str = std::string(str); + unsigned long value = strtoul(str_str.c_str(), &end_ptr, base); // NOLINT + if (value == ULONG_MAX && errno == ERANGE) + return false; // Value out of range for unsigned long. + if (sizeof(unsigned long) > sizeof(uint32_t) && value > 0xFFFFFFFF) // NOLINT + return false; // Value out of range for uint32_t. + if (end_ptr - str_str.c_str() < static_cast(str.length())) + return false; // Part of the string was not parsed. + *ssrc = static_cast(value); + return true; +} + +static bool ValidateExtensionId(int value) { + if (value > 0 && value <= 255) // Value is ok. + return true; + printf("Extension ID must be between 1 and 255, not %d\n", + static_cast(value)); + return false; +} + +// Flag validators. +bool ValidatePayloadType(int value) { + if (value >= 0 && value <= 127) // Value is ok. + return true; + printf("Payload type must be between 0 and 127, not %d\n", + static_cast(value)); + return false; +} + +bool ValidateSsrcValue(absl::string_view str) { + uint32_t dummy_ssrc; + if (ParseSsrc(str, &dummy_ssrc)) // Value is ok. + return true; + printf("Invalid SSRC: %.*s\n", static_cast(str.size()), str.data()); + return false; +} + +void PrintCodecMappingEntry(absl::string_view codec, int flag) { + std::cout << codec << ": " << flag << std::endl; +} + +void PrintCodecMapping() { + PrintCodecMappingEntry("PCM-u", absl::GetFlag(FLAGS_pcmu)); + PrintCodecMappingEntry("PCM-a", absl::GetFlag(FLAGS_pcma)); + PrintCodecMappingEntry("iLBC", absl::GetFlag(FLAGS_ilbc)); + PrintCodecMappingEntry("iSAC", absl::GetFlag(FLAGS_isac)); + PrintCodecMappingEntry("iSAC-swb (32 kHz)", absl::GetFlag(FLAGS_isac_swb)); + PrintCodecMappingEntry("Opus", absl::GetFlag(FLAGS_opus)); + PrintCodecMappingEntry("PCM16b-nb (8 kHz)", absl::GetFlag(FLAGS_pcm16b)); + PrintCodecMappingEntry("PCM16b-wb (16 kHz)", absl::GetFlag(FLAGS_pcm16b_wb)); + PrintCodecMappingEntry("PCM16b-swb32 (32 kHz)", + absl::GetFlag(FLAGS_pcm16b_swb32)); + PrintCodecMappingEntry("PCM16b-swb48 (48 kHz)", + absl::GetFlag(FLAGS_pcm16b_swb48)); + PrintCodecMappingEntry("G.722", absl::GetFlag(FLAGS_g722)); + PrintCodecMappingEntry("AVT/DTMF (8 kHz)", absl::GetFlag(FLAGS_avt)); + PrintCodecMappingEntry("AVT/DTMF (16 kHz)", absl::GetFlag(FLAGS_avt_16)); + PrintCodecMappingEntry("AVT/DTMF (32 kHz)", absl::GetFlag(FLAGS_avt_32)); + PrintCodecMappingEntry("AVT/DTMF (48 kHz)", absl::GetFlag(FLAGS_avt_48)); + PrintCodecMappingEntry("redundant audio (RED)", absl::GetFlag(FLAGS_red)); + PrintCodecMappingEntry("comfort noise (8 kHz)", absl::GetFlag(FLAGS_cn_nb)); + PrintCodecMappingEntry("comfort noise (16 kHz)", absl::GetFlag(FLAGS_cn_wb)); + PrintCodecMappingEntry("comfort noise (32 kHz)", + absl::GetFlag(FLAGS_cn_swb32)); + PrintCodecMappingEntry("comfort noise (48 kHz)", + absl::GetFlag(FLAGS_cn_swb48)); +} + +bool ValidateOutputFilesOptions(bool textlog, + bool plotting, + absl::string_view output_files_base_name, + absl::string_view output_audio_filename) { + bool output_files_base_name_specified = !output_files_base_name.empty(); + if (!textlog && !plotting && output_files_base_name_specified) { + std::cout << "Error: --output_files_base_name cannot be used without at " + "least one of the following flags: --textlog, --matlabplot, " + "--pythonplot." + << std::endl; + return false; + } + // Without `output_audio_filename`, `output_files_base_name` is required when + // plotting output files must be generated (in order to form a valid output + // file name). + if (output_audio_filename.empty() && plotting && + !output_files_base_name_specified) { + std::cout << "Error: when no output audio file is specified and " + "--matlabplot and/or --pythonplot are used, " + "--output_files_base_name must be also used." + << std::endl; + return false; + } + return true; +} + +absl::optional CreateOptionalOutputFileName( + bool output_requested, + absl::string_view basename, + absl::string_view output_audio_filename, + absl::string_view suffix) { + if (!output_requested) { + return absl::nullopt; + } + if (!basename.empty()) { + // Override the automatic assignment. + rtc::StringBuilder sb(basename); + sb << suffix; + return sb.str(); + } + if (!output_audio_filename.empty()) { + // Automatically assign name. + rtc::StringBuilder sb(output_audio_filename); + sb << suffix; + return sb.str(); + } + std::cout << "Error: invalid text log file parameters."; + return absl::nullopt; +} + +} // namespace + +int main(int argc, char* argv[]) { + std::vector args = absl::ParseCommandLine(argc, argv); + webrtc::test::NetEqTestFactory factory; + std::string usage = + "Tool for decoding an RTP dump file using NetEq.\n" + "Example usage:\n" + "./neteq_rtpplay input.rtp [output.{pcm, wav}]\n"; + if (absl::GetFlag(FLAGS_codec_map)) { + PrintCodecMapping(); + exit(0); + } + if (args.size() != 2 && + args.size() != 3) { // The output audio file is optional. + // Print usage information. + std::cout << usage; + exit(0); + } + const std::string output_audio_filename((args.size() == 3) ? args[2] : ""); + const std::string output_files_base_name( + absl::GetFlag(FLAGS_output_files_base_name)); + RTC_CHECK(ValidateOutputFilesOptions( + absl::GetFlag(FLAGS_textlog), + absl::GetFlag(FLAGS_matlabplot) || absl::GetFlag(FLAGS_pythonplot), + output_files_base_name, output_audio_filename)); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_pcmu))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_pcma))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_ilbc))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_isac))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_isac_swb))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_opus))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_pcm16b))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_pcm16b_wb))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_pcm16b_swb32))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_pcm16b_swb48))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_g722))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_avt))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_avt_16))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_avt_32))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_avt_48))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_red))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_cn_nb))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_cn_wb))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_cn_swb32))); + RTC_CHECK(ValidatePayloadType(absl::GetFlag(FLAGS_cn_swb48))); + RTC_CHECK(ValidateSsrcValue(absl::GetFlag(FLAGS_ssrc))); + RTC_CHECK(ValidateExtensionId(absl::GetFlag(FLAGS_audio_level))); + RTC_CHECK(ValidateExtensionId(absl::GetFlag(FLAGS_abs_send_time))); + RTC_CHECK(ValidateExtensionId(absl::GetFlag(FLAGS_transport_seq_no))); + RTC_CHECK(ValidateExtensionId(absl::GetFlag(FLAGS_video_content_type))); + RTC_CHECK(ValidateExtensionId(absl::GetFlag(FLAGS_video_timing))); + + // Make force_fieldtrials persistent string during entire program live as + // absl::GetFlag creates temporary string and c_str() will point to + // deallocated string. + const std::string force_fieldtrials = absl::GetFlag(FLAGS_force_fieldtrials); + webrtc::field_trial::InitFieldTrialsFromString(force_fieldtrials.c_str()); + + webrtc::test::NetEqTestFactory::Config config; + config.pcmu = absl::GetFlag(FLAGS_pcmu); + config.pcma = absl::GetFlag(FLAGS_pcma); + config.ilbc = absl::GetFlag(FLAGS_ilbc); + config.isac = absl::GetFlag(FLAGS_isac); + config.isac_swb = absl::GetFlag(FLAGS_isac_swb); + config.opus = absl::GetFlag(FLAGS_opus); + config.pcm16b = absl::GetFlag(FLAGS_pcm16b); + config.pcm16b_wb = absl::GetFlag(FLAGS_pcm16b_wb); + config.pcm16b_swb32 = absl::GetFlag(FLAGS_pcm16b_swb32); + config.pcm16b_swb48 = absl::GetFlag(FLAGS_pcm16b_swb48); + config.g722 = absl::GetFlag(FLAGS_g722); + config.avt = absl::GetFlag(FLAGS_avt); + config.avt_16 = absl::GetFlag(FLAGS_avt_16); + config.avt_32 = absl::GetFlag(FLAGS_avt_32); + config.avt_48 = absl::GetFlag(FLAGS_avt_48); + config.red = absl::GetFlag(FLAGS_red); + config.cn_nb = absl::GetFlag(FLAGS_cn_nb); + config.cn_wb = absl::GetFlag(FLAGS_cn_wb); + config.cn_swb32 = absl::GetFlag(FLAGS_cn_swb32); + config.cn_swb48 = absl::GetFlag(FLAGS_cn_swb48); + config.replacement_audio_file = absl::GetFlag(FLAGS_replacement_audio_file); + config.audio_level = absl::GetFlag(FLAGS_audio_level); + config.abs_send_time = absl::GetFlag(FLAGS_abs_send_time); + config.transport_seq_no = absl::GetFlag(FLAGS_transport_seq_no); + config.video_content_type = absl::GetFlag(FLAGS_video_content_type); + config.video_timing = absl::GetFlag(FLAGS_video_timing); + config.matlabplot = absl::GetFlag(FLAGS_matlabplot); + config.pythonplot = absl::GetFlag(FLAGS_pythonplot); + config.concealment_events = absl::GetFlag(FLAGS_concealment_events); + config.max_nr_packets_in_buffer = + absl::GetFlag(FLAGS_max_nr_packets_in_buffer); + config.enable_fast_accelerate = absl::GetFlag(FLAGS_enable_fast_accelerate); + if (!output_audio_filename.empty()) { + config.output_audio_filename = output_audio_filename; + } + config.textlog = absl::GetFlag(FLAGS_textlog); + config.textlog_filename = CreateOptionalOutputFileName( + absl::GetFlag(FLAGS_textlog), output_files_base_name, + output_audio_filename, ".text_log.txt"); + config.plot_scripts_basename = CreateOptionalOutputFileName( + absl::GetFlag(FLAGS_matlabplot) || absl::GetFlag(FLAGS_pythonplot), + output_files_base_name, output_audio_filename, ""); + + // Check if an SSRC value was provided. + if (absl::GetFlag(FLAGS_ssrc).size() > 0) { + uint32_t ssrc; + RTC_CHECK(ParseSsrc(absl::GetFlag(FLAGS_ssrc), &ssrc)) + << "Flag verification has failed."; + config.ssrc_filter = absl::make_optional(ssrc); + } + + std::unique_ptr test = + factory.InitializeTestFromFile(/*input_filename=*/args[1], + /*factory=*/nullptr, config); + RTC_CHECK(test) << "ERROR: Unable to run test"; + test->Run(); + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_rtpplay_test.sh b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_rtpplay_test.sh new file mode 100755 index 0000000000..0a6bf16016 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_rtpplay_test.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +# + +# Aliases. +BIN=$1 +TEST_RTC_EVENT_LOG=$2 +INPUT_PCM_FILE=$3 + +# Check setup. +if [ ! -f $BIN ]; then + echo "Cannot find neteq_rtpplay binary." + exit 99 +fi +if [ ! -f $TEST_RTC_EVENT_LOG ]; then + echo "Cannot find RTC event log file." + exit 99 +fi +if [ ! -f $INPUT_PCM_FILE ]; then + echo "Cannot find PCM file." + exit 99 +fi + +# Defines. + +TMP_DIR=$(mktemp -d /tmp/tmp_XXXXXXXXXX) +PASS=0 +FAIL=1 +TEST_SUITE_RESULT=$PASS + +file_hash () { + md5sum $1 | awk '{ print $1 }' +} + +test_passed () { + echo PASS +} + +test_failed () { + echo "FAIL: $1" + TEST_SUITE_RESULT=$FAIL +} + +test_file_checksums_match () { + if [ ! -f $1 ] || [ ! -f $2 ]; then + test_failed "Cannot compare hash values: file(s) not found." + return + fi + HASH1=$(file_hash $1) + HASH2=$(file_hash $2) + if [ "$HASH1" = "$HASH2" ]; then + test_passed + else + test_failed "$1 differs from $2" + fi +} + +test_file_exists () { + if [ -f $1 ]; then + test_passed + else + test_failed "$1 does not exist" + fi +} + +test_exit_code_0 () { + if [ $1 -eq 0 ]; then + test_passed + else + test_failed "$1 did not return 0" + fi +} + +test_exit_code_not_0 () { + if [ $1 -eq 0 ]; then + test_failed "$1 returned 0" + else + test_passed + fi +} + +# Generate test data. + +# Case 1. Pre-existing way. +CASE1_WAV=$TMP_DIR/case1.wav +$BIN $TEST_RTC_EVENT_LOG $CASE1_WAV \ + --replacement_audio_file $INPUT_PCM_FILE \ + --textlog --pythonplot --matlabplot \ + > $TMP_DIR/case1.stdout 2> /dev/null +CASE1_RETURN_CODE=$? +CASE1_TEXTLOG=$TMP_DIR/case1.wav.text_log.txt +CASE1_PYPLOT=$TMP_DIR/case1_wav.py +CASE1_MATPLOT=$TMP_DIR/case1_wav.m + +# Case 2. No output files. +$BIN $TEST_RTC_EVENT_LOG --replacement_audio_file $INPUT_PCM_FILE \ + > $TMP_DIR/case2.stdout 2> /dev/null +CASE2_RETURN_CODE=$? + +# Case 3. No output audio file. + +# Case 3.1 Without --output_files_base_name (won't run). +$BIN $TEST_RTC_EVENT_LOG \ + --replacement_audio_file $INPUT_PCM_FILE \ + --textlog --pythonplot --matlabplot \ + &> /dev/null +CASE3_1_RETURN_CODE=$? + +# Case 3.2 With --output_files_base_name (runs). +$BIN $TEST_RTC_EVENT_LOG \ + --replacement_audio_file $INPUT_PCM_FILE \ + --output_files_base_name $TMP_DIR/case3_2 \ + --textlog --pythonplot --matlabplot \ + > $TMP_DIR/case3_2.stdout 2> /dev/null +CASE3_2_RETURN_CODE=$? +CASE3_2_TEXTLOG=$TMP_DIR/case3_2.text_log.txt +CASE3_2_PYPLOT=$TMP_DIR/case3_2.py +CASE3_2_MATPLOT=$TMP_DIR/case3_2.m + +# Case 4. With output audio file and --output_files_base_name. +CASE4_WAV=$TMP_DIR/case4.wav +$BIN $TEST_RTC_EVENT_LOG $TMP_DIR/case4.wav \ + --replacement_audio_file $INPUT_PCM_FILE \ + --output_files_base_name $TMP_DIR/case4 \ + --textlog --pythonplot --matlabplot \ + > $TMP_DIR/case4.stdout 2> /dev/null +CASE4_RETURN_CODE=$? +CASE4_TEXTLOG=$TMP_DIR/case4.text_log.txt +CASE4_PYPLOT=$TMP_DIR/case4.py +CASE4_MATPLOT=$TMP_DIR/case4.m + +# Tests. + +echo Check exit codes +test_exit_code_0 $CASE1_RETURN_CODE +test_exit_code_0 $CASE2_RETURN_CODE +test_exit_code_not_0 $CASE3_1_RETURN_CODE +test_exit_code_0 $CASE3_2_RETURN_CODE +test_exit_code_0 $CASE4_RETURN_CODE + +echo Check that the expected output files exist +test_file_exists $CASE1_TEXTLOG +test_file_exists $CASE3_2_TEXTLOG +test_file_exists $CASE4_TEXTLOG +test_file_exists $CASE1_PYPLOT +test_file_exists $CASE3_2_PYPLOT +test_file_exists $CASE4_PYPLOT +test_file_exists $CASE1_MATPLOT +test_file_exists $CASE3_2_MATPLOT +test_file_exists $CASE4_MATPLOT + +echo Check that the same WAV file is produced +test_file_checksums_match $CASE1_WAV $CASE4_WAV + +echo Check that the same text log is produced +test_file_checksums_match $CASE1_TEXTLOG $CASE3_2_TEXTLOG +test_file_checksums_match $CASE1_TEXTLOG $CASE4_TEXTLOG + +echo Check that the same python plot scripts is produced +test_file_checksums_match $CASE1_PYPLOT $CASE3_2_PYPLOT +test_file_checksums_match $CASE1_PYPLOT $CASE4_PYPLOT + +echo Check that the same matlab plot scripts is produced +test_file_checksums_match $CASE1_MATPLOT $CASE3_2_MATPLOT +test_file_checksums_match $CASE1_MATPLOT $CASE4_MATPLOT + +# Clean up +rm -fr $TMP_DIR + +if [ $TEST_SUITE_RESULT -eq $PASS ]; then + echo All tests passed. + exit 0 +else + echo One or more tests failed. + exit 1 +fi diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_getter.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_getter.cc new file mode 100644 index 0000000000..6738e494f6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_getter.cc @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_stats_getter.h" + +#include +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { +namespace test { + +std::string NetEqStatsGetter::ConcealmentEvent::ToString() const { + char ss_buf[256]; + rtc::SimpleStringBuilder ss(ss_buf); + ss << "ConcealmentEvent duration_ms:" << duration_ms + << " event_number:" << concealment_event_number + << " time_from_previous_event_end_ms:" << time_from_previous_event_end_ms; + return ss.str(); +} + +NetEqStatsGetter::NetEqStatsGetter( + std::unique_ptr delay_analyzer) + : delay_analyzer_(std::move(delay_analyzer)) {} + +void NetEqStatsGetter::BeforeGetAudio(NetEq* neteq) { + if (delay_analyzer_) { + delay_analyzer_->BeforeGetAudio(neteq); + } +} + +void NetEqStatsGetter::AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool muted, + NetEq* neteq) { + // TODO(minyue): Get stats should better not be called as a call back after + // get audio. It is called independently from get audio in practice. + const auto lifetime_stat = neteq->GetLifetimeStatistics(); + if (last_stats_query_time_ms_ == 0 || + rtc::TimeDiff(time_now_ms, last_stats_query_time_ms_) >= + stats_query_interval_ms_) { + NetEqNetworkStatistics stats; + RTC_CHECK_EQ(neteq->NetworkStatistics(&stats), 0); + stats_.push_back(std::make_pair(time_now_ms, stats)); + lifetime_stats_.push_back(std::make_pair(time_now_ms, lifetime_stat)); + last_stats_query_time_ms_ = time_now_ms; + } + + const auto voice_concealed_samples = + lifetime_stat.concealed_samples - lifetime_stat.silent_concealed_samples; + if (current_concealment_event_ != lifetime_stat.concealment_events && + voice_concealed_samples_until_last_event_ < voice_concealed_samples) { + if (last_event_end_time_ms_ > 0) { + // Do not account for the first event to avoid start of the call + // skewing. + ConcealmentEvent concealment_event; + uint64_t last_event_voice_concealed_samples = + voice_concealed_samples - voice_concealed_samples_until_last_event_; + RTC_CHECK_GT(last_event_voice_concealed_samples, 0); + concealment_event.duration_ms = last_event_voice_concealed_samples / + (audio_frame.sample_rate_hz_ / 1000); + concealment_event.concealment_event_number = current_concealment_event_; + concealment_event.time_from_previous_event_end_ms = + time_now_ms - last_event_end_time_ms_; + concealment_events_.emplace_back(concealment_event); + voice_concealed_samples_until_last_event_ = voice_concealed_samples; + } + last_event_end_time_ms_ = time_now_ms; + voice_concealed_samples_until_last_event_ = voice_concealed_samples; + current_concealment_event_ = lifetime_stat.concealment_events; + } + + if (delay_analyzer_) { + delay_analyzer_->AfterGetAudio(time_now_ms, audio_frame, muted, neteq); + } +} + +double NetEqStatsGetter::AverageSpeechExpandRate() const { + double sum_speech_expand = std::accumulate( + stats_.begin(), stats_.end(), double{0.0}, + [](double a, std::pair b) { + return a + static_cast(b.second.speech_expand_rate); + }); + return sum_speech_expand / 16384.0 / stats_.size(); +} + +NetEqStatsGetter::Stats NetEqStatsGetter::AverageStats() const { + Stats sum_stats = std::accumulate( + stats_.begin(), stats_.end(), Stats(), + [](Stats a, std::pair bb) { + const auto& b = bb.second; + a.current_buffer_size_ms += b.current_buffer_size_ms; + a.preferred_buffer_size_ms += b.preferred_buffer_size_ms; + a.jitter_peaks_found += b.jitter_peaks_found; + a.expand_rate += b.expand_rate / 16384.0; + a.speech_expand_rate += b.speech_expand_rate / 16384.0; + a.preemptive_rate += b.preemptive_rate / 16384.0; + a.accelerate_rate += b.accelerate_rate / 16384.0; + a.secondary_decoded_rate += b.secondary_decoded_rate / 16384.0; + a.secondary_discarded_rate += b.secondary_discarded_rate / 16384.0; + a.mean_waiting_time_ms += b.mean_waiting_time_ms; + a.median_waiting_time_ms += b.median_waiting_time_ms; + a.min_waiting_time_ms = std::min( + a.min_waiting_time_ms, static_cast(b.min_waiting_time_ms)); + a.max_waiting_time_ms = std::max( + a.max_waiting_time_ms, static_cast(b.max_waiting_time_ms)); + return a; + }); + + sum_stats.current_buffer_size_ms /= stats_.size(); + sum_stats.preferred_buffer_size_ms /= stats_.size(); + sum_stats.jitter_peaks_found /= stats_.size(); + sum_stats.packet_loss_rate /= stats_.size(); + sum_stats.expand_rate /= stats_.size(); + sum_stats.speech_expand_rate /= stats_.size(); + sum_stats.preemptive_rate /= stats_.size(); + sum_stats.accelerate_rate /= stats_.size(); + sum_stats.secondary_decoded_rate /= stats_.size(); + sum_stats.secondary_discarded_rate /= stats_.size(); + sum_stats.added_zero_samples /= stats_.size(); + sum_stats.mean_waiting_time_ms /= stats_.size(); + sum_stats.median_waiting_time_ms /= stats_.size(); + + return sum_stats; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_getter.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_getter.h new file mode 100644 index 0000000000..b1b12bb1f8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_getter.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_GETTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_GETTER_H_ + +#include +#include +#include + +#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" + +namespace webrtc { +namespace test { + +class NetEqStatsGetter : public NetEqGetAudioCallback { + public: + // This struct is a replica of webrtc::NetEqNetworkStatistics, but with all + // values stored in double precision. + struct Stats { + double current_buffer_size_ms = 0.0; + double preferred_buffer_size_ms = 0.0; + double jitter_peaks_found = 0.0; + double packet_loss_rate = 0.0; + double expand_rate = 0.0; + double speech_expand_rate = 0.0; + double preemptive_rate = 0.0; + double accelerate_rate = 0.0; + double secondary_decoded_rate = 0.0; + double secondary_discarded_rate = 0.0; + double clockdrift_ppm = 0.0; + double added_zero_samples = 0.0; + double mean_waiting_time_ms = 0.0; + double median_waiting_time_ms = 0.0; + double min_waiting_time_ms = 0.0; + double max_waiting_time_ms = 0.0; + }; + + struct ConcealmentEvent { + uint64_t duration_ms; + size_t concealment_event_number; + int64_t time_from_previous_event_end_ms; + std::string ToString() const; + }; + + // Takes a pointer to another callback object, which will be invoked after + // this object finishes. This does not transfer ownership, and null is a + // valid value. + explicit NetEqStatsGetter(std::unique_ptr delay_analyzer); + + void set_stats_query_interval_ms(int64_t stats_query_interval_ms) { + stats_query_interval_ms_ = stats_query_interval_ms; + } + + void BeforeGetAudio(NetEq* neteq) override; + + void AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool muted, + NetEq* neteq) override; + + double AverageSpeechExpandRate() const; + + NetEqDelayAnalyzer* delay_analyzer() const { return delay_analyzer_.get(); } + + const std::vector& concealment_events() const { + // Do not account for the last concealment event to avoid potential end + // call skewing. + return concealment_events_; + } + + const std::vector>* stats() const { + return &stats_; + } + + const std::vector>* + lifetime_stats() const { + return &lifetime_stats_; + } + + Stats AverageStats() const; + + private: + std::unique_ptr delay_analyzer_; + int64_t stats_query_interval_ms_ = 1000; + int64_t last_stats_query_time_ms_ = 0; + std::vector> stats_; + std::vector> lifetime_stats_; + size_t current_concealment_event_ = 1; + uint64_t voice_concealed_samples_until_last_event_ = 0; + std::vector concealment_events_; + int64_t last_event_end_time_ms_ = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_GETTER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc new file mode 100644 index 0000000000..162a4c9300 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_stats_plotter.h" + +#include +#include + +#include + +#include "absl/strings/string_view.h" + +namespace webrtc { +namespace test { + +NetEqStatsPlotter::NetEqStatsPlotter(bool make_matlab_plot, + bool make_python_plot, + bool show_concealment_events, + absl::string_view base_file_name) + : make_matlab_plot_(make_matlab_plot), + make_python_plot_(make_python_plot), + show_concealment_events_(show_concealment_events), + base_file_name_(base_file_name) { + std::unique_ptr delay_analyzer; + if (make_matlab_plot || make_python_plot) { + delay_analyzer.reset(new NetEqDelayAnalyzer); + } + stats_getter_.reset(new NetEqStatsGetter(std::move(delay_analyzer))); +} + +void NetEqStatsPlotter::SimulationEnded(int64_t simulation_time_ms) { + if (make_matlab_plot_) { + auto matlab_script_name = base_file_name_; + std::replace(matlab_script_name.begin(), matlab_script_name.end(), '.', + '_'); + printf("Creating Matlab plot script %s.m\n", matlab_script_name.c_str()); + stats_getter_->delay_analyzer()->CreateMatlabScript(matlab_script_name + + ".m"); + } + if (make_python_plot_) { + auto python_script_name = base_file_name_; + std::replace(python_script_name.begin(), python_script_name.end(), '.', + '_'); + printf("Creating Python plot script %s.py\n", python_script_name.c_str()); + stats_getter_->delay_analyzer()->CreatePythonScript(python_script_name + + ".py"); + } + + printf("Simulation statistics:\n"); + printf(" output duration: %" PRId64 " ms\n", simulation_time_ms); + auto stats = stats_getter_->AverageStats(); + printf(" packet_loss_rate: %f %%\n", 100.0 * stats.packet_loss_rate); + printf(" expand_rate: %f %%\n", 100.0 * stats.expand_rate); + printf(" speech_expand_rate: %f %%\n", 100.0 * stats.speech_expand_rate); + printf(" preemptive_rate: %f %%\n", 100.0 * stats.preemptive_rate); + printf(" accelerate_rate: %f %%\n", 100.0 * stats.accelerate_rate); + printf(" secondary_decoded_rate: %f %%\n", + 100.0 * stats.secondary_decoded_rate); + printf(" secondary_discarded_rate: %f %%\n", + 100.0 * stats.secondary_discarded_rate); + printf(" clockdrift_ppm: %f ppm\n", stats.clockdrift_ppm); + printf(" mean_waiting_time_ms: %f ms\n", stats.mean_waiting_time_ms); + printf(" median_waiting_time_ms: %f ms\n", stats.median_waiting_time_ms); + printf(" min_waiting_time_ms: %f ms\n", stats.min_waiting_time_ms); + printf(" max_waiting_time_ms: %f ms\n", stats.max_waiting_time_ms); + printf(" current_buffer_size_ms: %f ms\n", stats.current_buffer_size_ms); + printf(" preferred_buffer_size_ms: %f ms\n", stats.preferred_buffer_size_ms); + if (show_concealment_events_) { + printf(" concealment_events_ms:\n"); + for (auto concealment_event : stats_getter_->concealment_events()) + printf("%s\n", concealment_event.ToString().c_str()); + printf(" end of concealment_events_ms\n"); + } + + const auto lifetime_stats_vector = stats_getter_->lifetime_stats(); + if (!lifetime_stats_vector->empty()) { + auto lifetime_stats = lifetime_stats_vector->back().second; + printf(" total_samples_received: %" PRIu64 "\n", + lifetime_stats.total_samples_received); + printf(" concealed_samples: %" PRIu64 "\n", + lifetime_stats.concealed_samples); + printf(" concealment_events: %" PRIu64 "\n", + lifetime_stats.concealment_events); + printf(" delayed_packet_outage_samples: %" PRIu64 "\n", + lifetime_stats.delayed_packet_outage_samples); + printf(" num_interruptions: %d\n", lifetime_stats.interruption_count); + printf(" sum_interruption_length_ms: %d ms\n", + lifetime_stats.total_interruption_duration_ms); + printf(" interruption_ratio: %f\n", + static_cast(lifetime_stats.total_interruption_duration_ms) / + simulation_time_ms); + printf(" removed_samples_for_acceleration: %" PRIu64 "\n", + lifetime_stats.removed_samples_for_acceleration); + printf(" inserted_samples_for_deceleration: %" PRIu64 "\n", + lifetime_stats.inserted_samples_for_deceleration); + printf(" generated_noise_samples: %" PRIu64 "\n", + lifetime_stats.generated_noise_samples); + printf(" packets_discarded: %" PRIu64 "\n", + lifetime_stats.packets_discarded); + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.h new file mode 100644 index 0000000000..11c16da9d1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_PLOTTER_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_PLOTTER_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h" +#include "modules/audio_coding/neteq/tools/neteq_stats_getter.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" + +namespace webrtc { +namespace test { + +class NetEqStatsPlotter : public NetEqSimulationEndedCallback { + public: + NetEqStatsPlotter(bool make_matlab_plot, + bool make_python_plot, + bool show_concealment_events, + absl::string_view base_file_name); + + void SimulationEnded(int64_t simulation_time_ms) override; + + NetEqStatsGetter* stats_getter() { return stats_getter_.get(); } + + private: + std::unique_ptr stats_getter_; + const bool make_matlab_plot_; + const bool make_python_plot_; + const bool show_concealment_events_; + const std::string base_file_name_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_PLOTTER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test.cc new file mode 100644 index 0000000000..a567efe2de --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test.cc @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_test.h" + +#include +#include + +#include "modules/audio_coding/neteq/default_neteq_factory.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace test { +namespace { + +absl::optional ActionToOperations( + absl::optional a) { + if (!a) { + return absl::nullopt; + } + switch (*a) { + case NetEqSimulator::Action::kAccelerate: + return absl::make_optional(NetEq::Operation::kAccelerate); + case NetEqSimulator::Action::kExpand: + return absl::make_optional(NetEq::Operation::kExpand); + case NetEqSimulator::Action::kNormal: + return absl::make_optional(NetEq::Operation::kNormal); + case NetEqSimulator::Action::kPreemptiveExpand: + return absl::make_optional(NetEq::Operation::kPreemptiveExpand); + } +} + +std::unique_ptr CreateNetEq( + const NetEq::Config& config, + Clock* clock, + const rtc::scoped_refptr& decoder_factory) { + return DefaultNetEqFactory().CreateNetEq(config, decoder_factory, clock); +} + +} // namespace + +void DefaultNetEqTestErrorCallback::OnInsertPacketError( + const NetEqInput::PacketData& packet) { + std::cerr << "InsertPacket returned an error." << std::endl; + std::cerr << "Packet data: " << packet.ToString() << std::endl; + RTC_FATAL(); +} + +void DefaultNetEqTestErrorCallback::OnGetAudioError() { + std::cerr << "GetAudio returned an error." << std::endl; + RTC_FATAL(); +} + +NetEqTest::NetEqTest(const NetEq::Config& config, + rtc::scoped_refptr decoder_factory, + const DecoderMap& codecs, + std::unique_ptr text_log, + NetEqFactory* neteq_factory, + std::unique_ptr input, + std::unique_ptr output, + Callbacks callbacks) + : input_(std::move(input)), + clock_(Timestamp::Millis(input_->NextEventTime().value_or(0))), + neteq_(neteq_factory + ? neteq_factory->CreateNetEq(config, decoder_factory, &clock_) + : CreateNetEq(config, &clock_, decoder_factory)), + output_(std::move(output)), + callbacks_(callbacks), + sample_rate_hz_(config.sample_rate_hz), + text_log_(std::move(text_log)) { + RTC_CHECK(!config.enable_muted_state) + << "The code does not handle enable_muted_state"; + RegisterDecoders(codecs); +} + +NetEqTest::~NetEqTest() = default; + +int64_t NetEqTest::Run() { + int64_t simulation_time = 0; + SimulationStepResult step_result; + do { + step_result = RunToNextGetAudio(); + simulation_time += step_result.simulation_step_ms; + } while (!step_result.is_simulation_finished); + if (callbacks_.simulation_ended_callback) { + callbacks_.simulation_ended_callback->SimulationEnded(simulation_time); + } + return simulation_time; +} + +NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { + SimulationStepResult result; + const int64_t start_time_ms = *input_->NextEventTime(); + int64_t time_now_ms = clock_.CurrentTime().ms(); + current_state_.packet_iat_ms.clear(); + + while (!input_->ended()) { + // Advance time to next event. + RTC_DCHECK(input_->NextEventTime()); + clock_.AdvanceTimeMilliseconds(*input_->NextEventTime() - time_now_ms); + time_now_ms = *input_->NextEventTime(); + // Check if it is time to insert packet. + if (input_->NextPacketTime() && time_now_ms >= *input_->NextPacketTime()) { + std::unique_ptr packet_data = input_->PopPacket(); + RTC_CHECK(packet_data); + const size_t payload_data_length = + packet_data->payload.size() - packet_data->header.paddingLength; + if (payload_data_length != 0) { + int error = neteq_->InsertPacket( + packet_data->header, + rtc::ArrayView(packet_data->payload)); + if (error != NetEq::kOK && callbacks_.error_callback) { + callbacks_.error_callback->OnInsertPacketError(*packet_data); + } + if (callbacks_.post_insert_packet) { + callbacks_.post_insert_packet->AfterInsertPacket(*packet_data, + neteq_.get()); + } + } else { + neteq_->InsertEmptyPacket(packet_data->header); + } + if (last_packet_time_ms_) { + current_state_.packet_iat_ms.push_back(time_now_ms - + *last_packet_time_ms_); + } + if (text_log_) { + const auto ops_state = neteq_->GetOperationsAndState(); + const auto delta_wallclock = + last_packet_time_ms_ ? (time_now_ms - *last_packet_time_ms_) : -1; + const auto delta_timestamp = + last_packet_timestamp_ + ? (static_cast(packet_data->header.timestamp) - + *last_packet_timestamp_) * + 1000 / sample_rate_hz_ + : -1; + const auto packet_size_bytes = + packet_data->payload.size() == 12 + ? ByteReader::ReadLittleEndian( + &packet_data->payload[8]) + : -1; + *text_log_ << "Packet - wallclock: " << std::setw(5) << time_now_ms + << ", delta wc: " << std::setw(4) << delta_wallclock + << ", seq_no: " << packet_data->header.sequenceNumber + << ", timestamp: " << std::setw(10) + << packet_data->header.timestamp + << ", delta ts: " << std::setw(4) << delta_timestamp + << ", size: " << std::setw(5) << packet_size_bytes + << ", frame size: " << std::setw(3) + << ops_state.current_frame_size_ms + << ", buffer size: " << std::setw(4) + << ops_state.current_buffer_size_ms << std::endl; + } + last_packet_time_ms_ = absl::make_optional(time_now_ms); + last_packet_timestamp_ = + absl::make_optional(packet_data->header.timestamp); + } + + // Check if it is time to get output audio. + if (input_->NextOutputEventTime() && + time_now_ms >= *input_->NextOutputEventTime()) { + if (callbacks_.get_audio_callback) { + callbacks_.get_audio_callback->BeforeGetAudio(neteq_.get()); + } + AudioFrame out_frame; + bool muted; + int error = neteq_->GetAudio(&out_frame, &muted, nullptr, + ActionToOperations(next_action_)); + next_action_ = absl::nullopt; + RTC_CHECK(!muted) << "The code does not handle enable_muted_state"; + if (error != NetEq::kOK) { + if (callbacks_.error_callback) { + callbacks_.error_callback->OnGetAudioError(); + } + } else { + sample_rate_hz_ = out_frame.sample_rate_hz_; + } + if (callbacks_.get_audio_callback) { + callbacks_.get_audio_callback->AfterGetAudio(time_now_ms, out_frame, + muted, neteq_.get()); + } + + if (output_) { + RTC_CHECK(output_->WriteArray( + out_frame.data(), + out_frame.samples_per_channel_ * out_frame.num_channels_)); + } + + input_->AdvanceOutputEvent(); + result.simulation_step_ms = + input_->NextEventTime().value_or(time_now_ms) - start_time_ms; + const auto operations_state = neteq_->GetOperationsAndState(); + current_state_.current_delay_ms = operations_state.current_buffer_size_ms; + current_state_.packet_size_ms = operations_state.current_frame_size_ms; + current_state_.next_packet_available = + operations_state.next_packet_available; + current_state_.packet_buffer_flushed = + operations_state.packet_buffer_flushes > + prev_ops_state_.packet_buffer_flushes; + // TODO(ivoc): Add more accurate reporting by tracking the origin of + // samples in the sync buffer. + result.action_times_ms[Action::kExpand] = 0; + result.action_times_ms[Action::kAccelerate] = 0; + result.action_times_ms[Action::kPreemptiveExpand] = 0; + result.action_times_ms[Action::kNormal] = 0; + + if (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC || + out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG) { + // Consider the whole frame to be the result of expansion. + result.action_times_ms[Action::kExpand] = 10; + } else if (operations_state.accelerate_samples - + prev_ops_state_.accelerate_samples > + 0) { + // Consider the whole frame to be the result of acceleration. + result.action_times_ms[Action::kAccelerate] = 10; + } else if (operations_state.preemptive_samples - + prev_ops_state_.preemptive_samples > + 0) { + // Consider the whole frame to be the result of preemptive expansion. + result.action_times_ms[Action::kPreemptiveExpand] = 10; + } else { + // Consider the whole frame to be the result of normal playout. + result.action_times_ms[Action::kNormal] = 10; + } + auto lifetime_stats = LifetimeStats(); + if (text_log_) { + const bool plc = + (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC) || + (out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG); + const bool cng = out_frame.speech_type_ == AudioFrame::SpeechType::kCNG; + const bool voice_concealed = + (lifetime_stats.concealed_samples - + lifetime_stats.silent_concealed_samples) > + (prev_lifetime_stats_.concealed_samples - + prev_lifetime_stats_.silent_concealed_samples); + *text_log_ << "GetAudio - wallclock: " << std::setw(5) << time_now_ms + << ", delta wc: " << std::setw(4) + << (input_->NextEventTime().value_or(time_now_ms) - + start_time_ms) + << ", CNG: " << cng << ", PLC: " << plc + << ", voice concealed: " << voice_concealed + << ", buffer size: " << std::setw(4) + << current_state_.current_delay_ms << std::endl; + if (lifetime_stats.packets_discarded > + prev_lifetime_stats_.packets_discarded) { + *text_log_ << "Discarded " + << (lifetime_stats.packets_discarded - + prev_lifetime_stats_.packets_discarded) + << " primary packets." << std::endl; + } + if (operations_state.packet_buffer_flushes > + prev_ops_state_.packet_buffer_flushes) { + *text_log_ << "Flushed packet buffer " + << (operations_state.packet_buffer_flushes - + prev_ops_state_.packet_buffer_flushes) + << " times." << std::endl; + } + } + prev_lifetime_stats_ = lifetime_stats; + const bool no_more_packets_to_decode = + !input_->NextPacketTime() && !operations_state.next_packet_available; + // End the simulation if the gap is too large. This indicates an issue + // with the event log file. + const bool simulation_step_too_large = result.simulation_step_ms > 1000; + if (simulation_step_too_large) { + // If we don't reset the step time, the large gap will be included in + // the simulation time, which can be a large distortion. + result.simulation_step_ms = 10; + } + result.is_simulation_finished = simulation_step_too_large || + no_more_packets_to_decode || + input_->ended(); + prev_ops_state_ = operations_state; + return result; + } + } + result.simulation_step_ms = + input_->NextEventTime().value_or(time_now_ms) - start_time_ms; + result.is_simulation_finished = true; + return result; +} + +void NetEqTest::SetNextAction(NetEqTest::Action next_operation) { + next_action_ = absl::optional(next_operation); +} + +NetEqTest::NetEqState NetEqTest::GetNetEqState() { + return current_state_; +} + +NetEqNetworkStatistics NetEqTest::SimulationStats() { + NetEqNetworkStatistics stats; + RTC_CHECK_EQ(neteq_->NetworkStatistics(&stats), 0); + return stats; +} + +NetEqLifetimeStatistics NetEqTest::LifetimeStats() const { + return neteq_->GetLifetimeStatistics(); +} + +NetEqTest::DecoderMap NetEqTest::StandardDecoderMap() { + DecoderMap codecs = { + {0, SdpAudioFormat("pcmu", 8000, 1)}, + {8, SdpAudioFormat("pcma", 8000, 1)}, +#ifdef WEBRTC_CODEC_ILBC + {102, SdpAudioFormat("ilbc", 8000, 1)}, +#endif +#ifdef WEBRTC_CODEC_OPUS + {111, SdpAudioFormat("opus", 48000, 2)}, +#endif + {93, SdpAudioFormat("l16", 8000, 1)}, + {94, SdpAudioFormat("l16", 16000, 1)}, + {95, SdpAudioFormat("l16", 32000, 1)}, + {96, SdpAudioFormat("l16", 48000, 1)}, + {9, SdpAudioFormat("g722", 8000, 1)}, + {106, SdpAudioFormat("telephone-event", 8000, 1)}, + {114, SdpAudioFormat("telephone-event", 16000, 1)}, + {115, SdpAudioFormat("telephone-event", 32000, 1)}, + {116, SdpAudioFormat("telephone-event", 48000, 1)}, + {117, SdpAudioFormat("red", 8000, 1)}, + {13, SdpAudioFormat("cn", 8000, 1)}, + {98, SdpAudioFormat("cn", 16000, 1)}, + {99, SdpAudioFormat("cn", 32000, 1)}, + {100, SdpAudioFormat("cn", 48000, 1)} + }; + return codecs; +} + +void NetEqTest::RegisterDecoders(const DecoderMap& codecs) { + for (const auto& c : codecs) { + RTC_CHECK(neteq_->RegisterPayloadType(c.first, c.second)) + << "Cannot register " << c.second.name << " to payload type " + << c.first; + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test.h new file mode 100644 index 0000000000..1d3eeda453 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_H_ + +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/neteq/neteq.h" +#include "api/neteq/neteq_factory.h" +#include "api/test/neteq_simulator.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace test { + +class NetEqTestErrorCallback { + public: + virtual ~NetEqTestErrorCallback() = default; + virtual void OnInsertPacketError(const NetEqInput::PacketData& packet) {} + virtual void OnGetAudioError() {} +}; + +class DefaultNetEqTestErrorCallback : public NetEqTestErrorCallback { + void OnInsertPacketError(const NetEqInput::PacketData& packet) override; + void OnGetAudioError() override; +}; + +class NetEqPostInsertPacket { + public: + virtual ~NetEqPostInsertPacket() = default; + virtual void AfterInsertPacket(const NetEqInput::PacketData& packet, + NetEq* neteq) = 0; +}; + +class NetEqGetAudioCallback { + public: + virtual ~NetEqGetAudioCallback() = default; + virtual void BeforeGetAudio(NetEq* neteq) = 0; + virtual void AfterGetAudio(int64_t time_now_ms, + const AudioFrame& audio_frame, + bool muted, + NetEq* neteq) = 0; +}; + +class NetEqSimulationEndedCallback { + public: + virtual ~NetEqSimulationEndedCallback() = default; + virtual void SimulationEnded(int64_t simulation_time_ms) = 0; +}; + +// Class that provides an input--output test for NetEq. The input (both packets +// and output events) is provided by a NetEqInput object, while the output is +// directed to an AudioSink object. +class NetEqTest : public NetEqSimulator { + public: + using DecoderMap = std::map; + + struct Callbacks { + NetEqTestErrorCallback* error_callback = nullptr; + NetEqPostInsertPacket* post_insert_packet = nullptr; + NetEqGetAudioCallback* get_audio_callback = nullptr; + NetEqSimulationEndedCallback* simulation_ended_callback = nullptr; + }; + + // Sets up the test with given configuration, codec mappings, input, ouput, + // and callback objects for error reporting. + NetEqTest(const NetEq::Config& config, + rtc::scoped_refptr decoder_factory, + const DecoderMap& codecs, + std::unique_ptr text_log, + NetEqFactory* neteq_factory, + std::unique_ptr input, + std::unique_ptr output, + Callbacks callbacks); + + ~NetEqTest() override; + + // Runs the test. Returns the duration of the produced audio in ms. + int64_t Run() override; + // Runs the simulation until we hit the next GetAudio event. If the simulation + // is finished, is_simulation_finished will be set to true in the returned + // SimulationStepResult. + SimulationStepResult RunToNextGetAudio() override; + + void SetNextAction(Action next_operation) override; + NetEqState GetNetEqState() override; + + // Returns the statistics from NetEq. + NetEqNetworkStatistics SimulationStats(); + NetEqLifetimeStatistics LifetimeStats() const; + + static DecoderMap StandardDecoderMap(); + + private: + void RegisterDecoders(const DecoderMap& codecs); + std::unique_ptr input_; + SimulatedClock clock_; + absl::optional next_action_; + absl::optional last_packet_time_ms_; + std::unique_ptr neteq_; + std::unique_ptr output_; + Callbacks callbacks_; + int sample_rate_hz_; + NetEqState current_state_; + NetEqOperationsAndState prev_ops_state_; + NetEqLifetimeStatistics prev_lifetime_stats_; + absl::optional last_packet_timestamp_; + std::unique_ptr text_log_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test_factory.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test_factory.cc new file mode 100644 index 0000000000..6cd371406c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test_factory.cc @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/neteq_test_factory.h" + +#include +#include // For ULONG_MAX returned by strtoul. +#include +#include // For strtoul. + +#include +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/neteq/neteq.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" +#include "modules/audio_coding/neteq/tools/fake_decode_from_file.h" +#include "modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h" +#include "modules/audio_coding/neteq/tools/neteq_event_log_input.h" +#include "modules/audio_coding/neteq/tools/neteq_packet_source_input.h" +#include "modules/audio_coding/neteq/tools/neteq_replacement_input.h" +#include "modules/audio_coding/neteq/tools/neteq_stats_getter.h" +#include "modules/audio_coding/neteq/tools/neteq_stats_plotter.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" +#include "modules/audio_coding/neteq/tools/output_audio_file.h" +#include "modules/audio_coding/neteq/tools/output_wav_file.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" +#include "rtc_base/checks.h" +#include "test/function_audio_decoder_factory.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { +namespace { + +absl::optional CodecSampleRate( + uint8_t payload_type, + webrtc::test::NetEqTestFactory::Config config) { + if (payload_type == config.pcmu || payload_type == config.pcma || + payload_type == config.ilbc || payload_type == config.pcm16b || + payload_type == config.cn_nb || payload_type == config.avt) + return 8000; + if (payload_type == config.isac || payload_type == config.pcm16b_wb || + payload_type == config.g722 || payload_type == config.cn_wb || + payload_type == config.avt_16) + return 16000; + if (payload_type == config.isac_swb || payload_type == config.pcm16b_swb32 || + payload_type == config.cn_swb32 || payload_type == config.avt_32) + return 32000; + if (payload_type == config.opus || payload_type == config.pcm16b_swb48 || + payload_type == config.cn_swb48 || payload_type == config.avt_48) + return 48000; + if (payload_type == config.red) + return 0; + return absl::nullopt; +} + +} // namespace + +// A callback class which prints whenver the inserted packet stream changes +// the SSRC. +class SsrcSwitchDetector : public NetEqPostInsertPacket { + public: + // Takes a pointer to another callback object, which will be invoked after + // this object finishes. This does not transfer ownership, and null is a + // valid value. + explicit SsrcSwitchDetector(NetEqPostInsertPacket* other_callback) + : other_callback_(other_callback) {} + + void AfterInsertPacket(const NetEqInput::PacketData& packet, + NetEq* neteq) override { + if (last_ssrc_ && packet.header.ssrc != *last_ssrc_) { + std::cout << "Changing streams from 0x" << std::hex << *last_ssrc_ + << " to 0x" << std::hex << packet.header.ssrc << std::dec + << " (payload type " + << static_cast(packet.header.payloadType) << ")" + << std::endl; + } + last_ssrc_ = packet.header.ssrc; + if (other_callback_) { + other_callback_->AfterInsertPacket(packet, neteq); + } + } + + private: + NetEqPostInsertPacket* other_callback_; + absl::optional last_ssrc_; +}; + +NetEqTestFactory::NetEqTestFactory() = default; +NetEqTestFactory::~NetEqTestFactory() = default; + +NetEqTestFactory::Config::Config() = default; +NetEqTestFactory::Config::Config(const Config& other) = default; +NetEqTestFactory::Config::~Config() = default; + +std::unique_ptr NetEqTestFactory::InitializeTestFromString( + absl::string_view input_string, + NetEqFactory* factory, + const Config& config) { + std::unique_ptr input( + NetEqEventLogInput::CreateFromString(input_string, config.ssrc_filter)); + if (!input) { + std::cerr << "Error: Cannot parse input string" << std::endl; + return nullptr; + } + return InitializeTest(std::move(input), factory, config); +} + +std::unique_ptr NetEqTestFactory::InitializeTestFromFile( + absl::string_view input_file_name, + NetEqFactory* factory, + const Config& config) { + // Gather RTP header extensions in a map. + NetEqPacketSourceInput::RtpHeaderExtensionMap rtp_ext_map = { + {config.audio_level, kRtpExtensionAudioLevel}, + {config.abs_send_time, kRtpExtensionAbsoluteSendTime}, + {config.transport_seq_no, kRtpExtensionTransportSequenceNumber}, + {config.video_content_type, kRtpExtensionVideoContentType}, + {config.video_timing, kRtpExtensionVideoTiming}}; + + std::unique_ptr input; + if (RtpFileSource::ValidRtpDump(input_file_name) || + RtpFileSource::ValidPcap(input_file_name)) { + input.reset(new NetEqRtpDumpInput(input_file_name, rtp_ext_map, + config.ssrc_filter)); + } else { + input.reset(NetEqEventLogInput::CreateFromFile(input_file_name, + config.ssrc_filter)); + } + + std::cout << "Input file: " << input_file_name << std::endl; + if (!input) { + std::cerr << "Error: Cannot open input file" << std::endl; + return nullptr; + } + return InitializeTest(std::move(input), factory, config); +} + +std::unique_ptr NetEqTestFactory::InitializeTest( + std::unique_ptr input, + NetEqFactory* factory, + const Config& config) { + if (input->ended()) { + std::cerr << "Error: Input is empty" << std::endl; + return nullptr; + } + + if (!config.field_trial_string.empty()) { + field_trials_ = + std::make_unique(config.field_trial_string); + } + + // Skip some initial events/packets if requested. + if (config.skip_get_audio_events > 0) { + std::cout << "Skipping " << config.skip_get_audio_events + << " get_audio events" << std::endl; + if (!input->NextPacketTime() || !input->NextOutputEventTime()) { + std::cerr << "No events found" << std::endl; + return nullptr; + } + for (int i = 0; i < config.skip_get_audio_events; i++) { + input->AdvanceOutputEvent(); + if (!input->NextOutputEventTime()) { + std::cerr << "Not enough get_audio events found" << std::endl; + return nullptr; + } + } + while (*input->NextPacketTime() < *input->NextOutputEventTime()) { + input->PopPacket(); + if (!input->NextPacketTime()) { + std::cerr << "Not enough incoming packets found" << std::endl; + return nullptr; + } + } + } + + // Check the sample rate. + absl::optional sample_rate_hz; + std::set> discarded_pt_and_ssrc; + while (absl::optional first_rtp_header = input->NextHeader()) { + RTC_DCHECK(first_rtp_header); + sample_rate_hz = CodecSampleRate(first_rtp_header->payloadType, config); + if (sample_rate_hz) { + std::cout << "Found valid packet with payload type " + << static_cast(first_rtp_header->payloadType) + << " and SSRC 0x" << std::hex << first_rtp_header->ssrc + << std::dec << std::endl; + if (config.initial_dummy_packets > 0) { + std::cout << "Nr of initial dummy packets: " + << config.initial_dummy_packets << std::endl; + input = std::make_unique( + std::move(input), config.initial_dummy_packets, *sample_rate_hz); + } + break; + } + // Discard this packet and move to the next. Keep track of discarded payload + // types and SSRCs. + discarded_pt_and_ssrc.emplace(first_rtp_header->payloadType, + first_rtp_header->ssrc); + input->PopPacket(); + } + if (!discarded_pt_and_ssrc.empty()) { + std::cout << "Discarded initial packets with the following payload types " + "and SSRCs:" + << std::endl; + for (const auto& d : discarded_pt_and_ssrc) { + std::cout << "PT " << d.first << "; SSRC 0x" << std::hex + << static_cast(d.second) << std::dec << std::endl; + } + } + if (!sample_rate_hz) { + std::cerr << "Cannot find any packets with known payload types" + << std::endl; + return nullptr; + } + + // If an output file is requested, open it. + std::unique_ptr output; + if (!config.output_audio_filename.has_value()) { + output = std::make_unique(); + std::cout << "No output audio file" << std::endl; + } else if (config.output_audio_filename->size() >= 4 && + config.output_audio_filename->substr( + config.output_audio_filename->size() - 4) == ".wav") { + // Open a wav file with the known sample rate. + output = std::make_unique(*config.output_audio_filename, + *sample_rate_hz); + std::cout << "Output WAV file: " << *config.output_audio_filename + << std::endl; + } else { + // Open a pcm file. + output = std::make_unique(*config.output_audio_filename); + std::cout << "Output PCM file: " << *config.output_audio_filename + << std::endl; + } + + NetEqTest::DecoderMap codecs = NetEqTest::StandardDecoderMap(); + + rtc::scoped_refptr decoder_factory = + CreateBuiltinAudioDecoderFactory(); + + // Check if a replacement audio file was provided. + if (config.replacement_audio_file.size() > 0) { + // Find largest unused payload type. + int replacement_pt = 127; + while (codecs.find(replacement_pt) != codecs.end()) { + --replacement_pt; + if (replacement_pt <= 0) { + std::cerr << "Error: Unable to find available replacement payload type" + << std::endl; + return nullptr; + } + } + + auto std_set_int32_to_uint8 = [](const std::set& a) { + std::set b; + for (auto& x : a) { + b.insert(static_cast(x)); + } + return b; + }; + + std::set cn_types = std_set_int32_to_uint8( + {config.cn_nb, config.cn_wb, config.cn_swb32, config.cn_swb48}); + std::set forbidden_types = + std_set_int32_to_uint8({config.g722, config.red, config.avt, + config.avt_16, config.avt_32, config.avt_48}); + input.reset(new NetEqReplacementInput(std::move(input), replacement_pt, + cn_types, forbidden_types)); + + // Note that capture-by-copy implies that the lambda captures the value of + // decoder_factory before it's reassigned on the left-hand side. + decoder_factory = rtc::make_ref_counted( + [decoder_factory, config]( + const SdpAudioFormat& format, + absl::optional codec_pair_id) { + std::unique_ptr decoder = + decoder_factory->MakeAudioDecoder(format, codec_pair_id); + if (!decoder && format.name == "replacement") { + decoder = std::make_unique( + std::make_unique(config.replacement_audio_file), + format.clockrate_hz, format.num_channels > 1); + } + return decoder; + }); + + if (!codecs + .insert({replacement_pt, SdpAudioFormat("replacement", 48000, 1)}) + .second) { + std::cerr << "Error: Unable to insert replacement audio codec" + << std::endl; + return nullptr; + } + } + + // Create a text log output stream if needed. + std::unique_ptr text_log; + if (config.textlog && config.textlog_filename.has_value()) { + // Write to file. + text_log = std::make_unique(*config.textlog_filename); + } else if (config.textlog) { + // Print to stdout. + text_log = std::make_unique(); + text_log->basic_ios::rdbuf(std::cout.rdbuf()); + } + + NetEqTest::Callbacks callbacks; + stats_plotter_ = std::make_unique( + config.matlabplot, config.pythonplot, config.concealment_events, + config.plot_scripts_basename.value_or("")); + + ssrc_switch_detector_.reset( + new SsrcSwitchDetector(stats_plotter_->stats_getter()->delay_analyzer())); + callbacks.post_insert_packet = ssrc_switch_detector_.get(); + callbacks.get_audio_callback = stats_plotter_->stats_getter(); + callbacks.simulation_ended_callback = stats_plotter_.get(); + NetEq::Config neteq_config; + neteq_config.sample_rate_hz = *sample_rate_hz; + neteq_config.max_packets_in_buffer = config.max_nr_packets_in_buffer; + neteq_config.enable_fast_accelerate = config.enable_fast_accelerate; + return std::make_unique( + neteq_config, decoder_factory, codecs, std::move(text_log), factory, + std::move(input), std::move(output), callbacks); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test_factory.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test_factory.h new file mode 100644 index 0000000000..96ce0b4334 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/neteq_test_factory.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_FACTORY_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_FACTORY_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/tools/neteq_test.h" +#include "test/field_trial.h" + +namespace webrtc { +namespace test { + +class SsrcSwitchDetector; +class NetEqStatsGetter; +class NetEqStatsPlotter; + +// Note that the NetEqTestFactory needs to be alive when the NetEqTest object is +// used for a simulation. +class NetEqTestFactory { + public: + NetEqTestFactory(); + ~NetEqTestFactory(); + struct Config { + Config(); + Config(const Config& other); + ~Config(); + // RTP payload type for PCM-u. + static constexpr int default_pcmu() { return 0; } + int pcmu = default_pcmu(); + // RTP payload type for PCM-a. + static constexpr int default_pcma() { return 8; } + int pcma = default_pcma(); + // RTP payload type for iLBC. + static constexpr int default_ilbc() { return 102; } + int ilbc = default_ilbc(); + // RTP payload type for iSAC. + static constexpr int default_isac() { return 103; } + int isac = default_isac(); + // RTP payload type for iSAC-swb (32 kHz). + static constexpr int default_isac_swb() { return 104; } + int isac_swb = default_isac_swb(); + // RTP payload type for Opus. + static constexpr int default_opus() { return 111; } + int opus = default_opus(); + // RTP payload type for PCM16b-nb (8 kHz). + static constexpr int default_pcm16b() { return 93; } + int pcm16b = default_pcm16b(); + // RTP payload type for PCM16b-wb (16 kHz). + static constexpr int default_pcm16b_wb() { return 94; } + int pcm16b_wb = default_pcm16b_wb(); + // RTP payload type for PCM16b-swb32 (32 kHz). + static constexpr int default_pcm16b_swb32() { return 95; } + int pcm16b_swb32 = default_pcm16b_swb32(); + // RTP payload type for PCM16b-swb48 (48 kHz). + static constexpr int default_pcm16b_swb48() { return 96; } + int pcm16b_swb48 = default_pcm16b_swb48(); + // RTP payload type for G.722. + static constexpr int default_g722() { return 9; } + int g722 = default_g722(); + // RTP payload type for AVT/DTMF (8 kHz). + static constexpr int default_avt() { return 106; } + int avt = default_avt(); + // RTP payload type for AVT/DTMF (16 kHz). + static constexpr int default_avt_16() { return 114; } + int avt_16 = default_avt_16(); + // RTP payload type for AVT/DTMF (32 kHz). + static constexpr int default_avt_32() { return 115; } + int avt_32 = default_avt_32(); + // RTP payload type for AVT/DTMF (48 kHz). + static constexpr int default_avt_48() { return 116; } + int avt_48 = default_avt_48(); + // RTP payload type for redundant audio (RED). + static constexpr int default_red() { return 117; } + int red = default_red(); + // RTP payload type for comfort noise (8 kHz). + static constexpr int default_cn_nb() { return 13; } + int cn_nb = default_cn_nb(); + // RTP payload type for comfort noise (16 kHz). + static constexpr int default_cn_wb() { return 98; } + int cn_wb = default_cn_wb(); + // RTP payload type for comfort noise (32 kHz). + static constexpr int default_cn_swb32() { return 99; } + int cn_swb32 = default_cn_swb32(); + // RTP payload type for comfort noise (48 kHz). + static constexpr int default_cn_swb48() { return 100; } + int cn_swb48 = default_cn_swb48(); + // A PCM file that will be used to populate dummy RTP packets. + std::string replacement_audio_file; + // Only use packets with this SSRC. + absl::optional ssrc_filter; + // Extension ID for audio level (RFC 6464). + static constexpr int default_audio_level() { return 1; } + int audio_level = default_audio_level(); + // Extension ID for absolute sender time. + static constexpr int default_abs_send_time() { return 3; } + int abs_send_time = default_abs_send_time(); + // Extension ID for transport sequence number. + static constexpr int default_transport_seq_no() { return 5; } + int transport_seq_no = default_transport_seq_no(); + // Extension ID for video content type. + static constexpr int default_video_content_type() { return 7; } + int video_content_type = default_video_content_type(); + // Extension ID for video timing. + static constexpr int default_video_timing() { return 8; } + int video_timing = default_video_timing(); + // Generate a matlab script for plotting the delay profile. + bool matlabplot = false; + // Generates a python script for plotting the delay profile. + bool pythonplot = false; + // Prints concealment events. + bool concealment_events = false; + // Maximum allowed number of packets in the buffer. + static constexpr int default_max_nr_packets_in_buffer() { return 200; } + int max_nr_packets_in_buffer = default_max_nr_packets_in_buffer(); + // Number of dummy packets to put in the packet buffer at the start of the + // simulation. + static constexpr int default_initial_dummy_packets() { return 0; } + int initial_dummy_packets = default_initial_dummy_packets(); + // Number of getAudio events to skip at the start of the simulation. + static constexpr int default_skip_get_audio_events() { return 0; } + int skip_get_audio_events = default_skip_get_audio_events(); + // Enables jitter buffer fast accelerate. + bool enable_fast_accelerate = false; + // Dumps events that describes the simulation on a step-by-step basis. + bool textlog = false; + // If specified and `textlog` is true, the output of `textlog` is written to + // the specified file name. + absl::optional textlog_filename; + // Base name for the output script files for plotting the delay profile. + absl::optional plot_scripts_basename; + // Path to the output audio file. + absl::optional output_audio_filename; + // Field trials to use during the simulation. + std::string field_trial_string; + }; + + std::unique_ptr InitializeTestFromFile( + absl::string_view input_filename, + NetEqFactory* neteq_factory, + const Config& config); + std::unique_ptr InitializeTestFromString( + absl::string_view input_string, + NetEqFactory* neteq_factory, + const Config& config); + + private: + std::unique_ptr InitializeTest(std::unique_ptr input, + NetEqFactory* neteq_factory, + const Config& config); + std::unique_ptr ssrc_switch_detector_; + std::unique_ptr stats_plotter_; + // The field trials are stored in the test factory, because neteq_test is not + // in a testonly target, and therefore cannot use ScopedFieldTrials. + std::unique_ptr field_trials_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_TEST_FACTORY_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/output_audio_file.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/output_audio_file.h new file mode 100644 index 0000000000..25577fc882 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/output_audio_file.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_AUDIO_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_AUDIO_FILE_H_ + +#include + +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" + +namespace webrtc { +namespace test { + +class OutputAudioFile : public AudioSink { + public: + // Creates an OutputAudioFile, opening a file named `file_name` for writing. + // The file format is 16-bit signed host-endian PCM. + explicit OutputAudioFile(absl::string_view file_name) { + out_file_ = fopen(std::string(file_name).c_str(), "wb"); + } + + virtual ~OutputAudioFile() { + if (out_file_) + fclose(out_file_); + } + + OutputAudioFile(const OutputAudioFile&) = delete; + OutputAudioFile& operator=(const OutputAudioFile&) = delete; + + bool WriteArray(const int16_t* audio, size_t num_samples) override { + RTC_DCHECK(out_file_); + return fwrite(audio, sizeof(*audio), num_samples, out_file_) == num_samples; + } + + private: + FILE* out_file_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_AUDIO_FILE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/output_wav_file.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/output_wav_file.h new file mode 100644 index 0000000000..20eedfb554 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/output_wav_file.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_WAV_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_WAV_FILE_H_ + +#include + +#include "absl/strings/string_view.h" +#include "common_audio/wav_file.h" +#include "modules/audio_coding/neteq/tools/audio_sink.h" + +namespace webrtc { +namespace test { + +class OutputWavFile : public AudioSink { + public: + // Creates an OutputWavFile, opening a file named `file_name` for writing. + // The output file is a PCM encoded wav file. + OutputWavFile(absl::string_view file_name, + int sample_rate_hz, + int num_channels = 1) + : wav_writer_(file_name, sample_rate_hz, num_channels) {} + + OutputWavFile(const OutputWavFile&) = delete; + OutputWavFile& operator=(const OutputWavFile&) = delete; + + bool WriteArray(const int16_t* audio, size_t num_samples) override { + wav_writer_.WriteSamples(audio, num_samples); + return true; + } + + private: + WavWriter wav_writer_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_OUTPUT_WAV_FILE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet.cc new file mode 100644 index 0000000000..e540173f43 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet.cc @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/packet.h" + +#include "api/array_view.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "rtc_base/checks.h" +#include "rtc_base/copy_on_write_buffer.h" + +namespace webrtc { +namespace test { + +Packet::Packet(rtc::CopyOnWriteBuffer packet, + size_t virtual_packet_length_bytes, + double time_ms, + const RtpHeaderExtensionMap* extension_map) + : packet_(std::move(packet)), + virtual_packet_length_bytes_(virtual_packet_length_bytes), + time_ms_(time_ms), + valid_header_(ParseHeader(extension_map)) {} + +Packet::Packet(const RTPHeader& header, + size_t virtual_packet_length_bytes, + size_t virtual_payload_length_bytes, + double time_ms) + : header_(header), + virtual_packet_length_bytes_(virtual_packet_length_bytes), + virtual_payload_length_bytes_(virtual_payload_length_bytes), + time_ms_(time_ms), + valid_header_(true) {} + +Packet::~Packet() = default; + +bool Packet::ExtractRedHeaders(std::list* headers) const { + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |1| block PT | timestamp offset | block length | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |1| ... | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |0| block PT | + // +-+-+-+-+-+-+-+-+ + // + + const uint8_t* payload_ptr = payload(); + const uint8_t* payload_end_ptr = payload_ptr + payload_length_bytes(); + + // Find all RED headers with the extension bit set to 1. That is, all headers + // but the last one. + while ((payload_ptr < payload_end_ptr) && (*payload_ptr & 0x80)) { + RTPHeader* header = new RTPHeader; + CopyToHeader(header); + header->payloadType = payload_ptr[0] & 0x7F; + uint32_t offset = (payload_ptr[1] << 6) + ((payload_ptr[2] & 0xFC) >> 2); + header->timestamp -= offset; + headers->push_front(header); + payload_ptr += 4; + } + // Last header. + RTC_DCHECK_LT(payload_ptr, payload_end_ptr); + if (payload_ptr >= payload_end_ptr) { + return false; // Payload too short. + } + RTPHeader* header = new RTPHeader; + CopyToHeader(header); + header->payloadType = payload_ptr[0] & 0x7F; + headers->push_front(header); + return true; +} + +void Packet::DeleteRedHeaders(std::list* headers) { + while (!headers->empty()) { + delete headers->front(); + headers->pop_front(); + } +} + +bool Packet::ParseHeader(const RtpHeaderExtensionMap* extension_map) { + // Use RtpPacketReceived instead of RtpPacket because former already has a + // converter into legacy RTPHeader. + webrtc::RtpPacketReceived rtp_packet(extension_map); + + // Because of the special case of dummy packets that have padding marked in + // the RTP header, but do not have rtp payload with the padding size, handle + // padding manually. Regular RTP packet parser reports failure, but it is fine + // in this context. + bool padding = (packet_[0] & 0b0010'0000); + size_t padding_size = 0; + if (padding) { + // Clear the padding bit to prevent failure when rtp payload is omited. + rtc::CopyOnWriteBuffer packet(packet_); + packet.MutableData()[0] &= ~0b0010'0000; + if (!rtp_packet.Parse(std::move(packet))) { + return false; + } + if (rtp_packet.payload_size() > 0) { + padding_size = rtp_packet.data()[rtp_packet.size() - 1]; + } + if (padding_size > rtp_packet.payload_size()) { + return false; + } + } else { + if (!rtp_packet.Parse(packet_)) { + return false; + } + } + rtp_payload_ = rtc::MakeArrayView(packet_.data() + rtp_packet.headers_size(), + rtp_packet.payload_size() - padding_size); + rtp_packet.GetHeader(&header_); + + RTC_CHECK_GE(virtual_packet_length_bytes_, rtp_packet.size()); + RTC_DCHECK_GE(virtual_packet_length_bytes_, rtp_packet.headers_size()); + virtual_payload_length_bytes_ = + virtual_packet_length_bytes_ - rtp_packet.headers_size(); + return true; +} + +void Packet::CopyToHeader(RTPHeader* destination) const { + *destination = header_; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet.h new file mode 100644 index 0000000000..96710907df --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_H_ + +#include + +#include "api/array_view.h" +#include "api/rtp_headers.h" +#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "rtc_base/copy_on_write_buffer.h" + +namespace webrtc { +namespace test { + +// Class for handling RTP packets in test applications. +class Packet { + public: + // Creates a packet, with the packet payload (including header bytes) in + // `packet`. The `time_ms` is an extra time associated with this packet, + // typically used to denote arrival time. + // `virtual_packet_length_bytes` is typically used when reading RTP dump files + // that only contain the RTP headers, and no payload (a.k.a RTP dummy files or + // RTP light). The `virtual_packet_length_bytes` tells what size the packet + // had on wire, including the now discarded payload. + Packet(rtc::CopyOnWriteBuffer packet, + size_t virtual_packet_length_bytes, + double time_ms, + const RtpHeaderExtensionMap* extension_map = nullptr); + + Packet(rtc::CopyOnWriteBuffer packet, + double time_ms, + const RtpHeaderExtensionMap* extension_map = nullptr) + : Packet(packet, packet.size(), time_ms, extension_map) {} + + // Same as above, but creates the packet from an already parsed RTPHeader. + // This is typically used when reading RTP dump files that only contain the + // RTP headers, and no payload. The `virtual_packet_length_bytes` tells what + // size the packet had on wire, including the now discarded payload, + // The `virtual_payload_length_bytes` tells the size of the payload. + Packet(const RTPHeader& header, + size_t virtual_packet_length_bytes, + size_t virtual_payload_length_bytes, + double time_ms); + + virtual ~Packet(); + + Packet(const Packet&) = delete; + Packet& operator=(const Packet&) = delete; + + // Parses the first bytes of the RTP payload, interpreting them as RED headers + // according to RFC 2198. The headers will be inserted into `headers`. The + // caller of the method assumes ownership of the objects in the list, and + // must delete them properly. + bool ExtractRedHeaders(std::list* headers) const; + + // Deletes all RTPHeader objects in `headers`, but does not delete `headers` + // itself. + static void DeleteRedHeaders(std::list* headers); + + const uint8_t* payload() const { return rtp_payload_.data(); } + + size_t packet_length_bytes() const { return packet_.size(); } + + size_t payload_length_bytes() const { return rtp_payload_.size(); } + + size_t virtual_packet_length_bytes() const { + return virtual_packet_length_bytes_; + } + + size_t virtual_payload_length_bytes() const { + return virtual_payload_length_bytes_; + } + + const RTPHeader& header() const { return header_; } + + double time_ms() const { return time_ms_; } + bool valid_header() const { return valid_header_; } + + private: + bool ParseHeader(const RtpHeaderExtensionMap* extension_map); + void CopyToHeader(RTPHeader* destination) const; + + RTPHeader header_; + const rtc::CopyOnWriteBuffer packet_; + rtc::ArrayView rtp_payload_; // Empty for dummy RTP packets. + // Virtual lengths are used when parsing RTP header files (dummy RTP files). + const size_t virtual_packet_length_bytes_; + size_t virtual_payload_length_bytes_ = 0; + const double time_ms_; // Used to denote a packet's arrival time. + const bool valid_header_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_source.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_source.cc new file mode 100644 index 0000000000..598ae6edd4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_source.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/packet_source.h" + +namespace webrtc { +namespace test { + +PacketSource::PacketSource() = default; + +PacketSource::~PacketSource() = default; + +void PacketSource::FilterOutPayloadType(uint8_t payload_type) { + filter_.set(payload_type, true); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_source.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_source.h new file mode 100644 index 0000000000..be1705cae1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_source.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_SOURCE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_SOURCE_H_ + +#include +#include + +#include "modules/audio_coding/neteq/tools/packet.h" + +namespace webrtc { +namespace test { + +// Interface class for an object delivering RTP packets to test applications. +class PacketSource { + public: + PacketSource(); + virtual ~PacketSource(); + + PacketSource(const PacketSource&) = delete; + PacketSource& operator=(const PacketSource&) = delete; + + // Returns next packet. Returns nullptr if the source is depleted, or if an + // error occurred. + virtual std::unique_ptr NextPacket() = 0; + + virtual void FilterOutPayloadType(uint8_t payload_type); + + protected: + std::bitset<128> filter_; // Payload type is 7 bits in the RFC. +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_PACKET_SOURCE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_unittest.cc new file mode 100644 index 0000000000..69cf56b529 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/packet_unittest.cc @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Unit tests for test Packet class. + +#include "modules/audio_coding/neteq/tools/packet.h" + +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +const int kHeaderLengthBytes = 12; + +void MakeRtpHeader(int payload_type, + int seq_number, + uint32_t timestamp, + uint32_t ssrc, + uint8_t* rtp_data) { + rtp_data[0] = 0x80; + rtp_data[1] = static_cast(payload_type); + rtp_data[2] = (seq_number >> 8) & 0xFF; + rtp_data[3] = (seq_number)&0xFF; + rtp_data[4] = timestamp >> 24; + rtp_data[5] = (timestamp >> 16) & 0xFF; + rtp_data[6] = (timestamp >> 8) & 0xFF; + rtp_data[7] = timestamp & 0xFF; + rtp_data[8] = ssrc >> 24; + rtp_data[9] = (ssrc >> 16) & 0xFF; + rtp_data[10] = (ssrc >> 8) & 0xFF; + rtp_data[11] = ssrc & 0xFF; +} +} // namespace + +TEST(TestPacket, RegularPacket) { + const size_t kPacketLengthBytes = 100; + rtc::CopyOnWriteBuffer packet_memory(kPacketLengthBytes); + const uint8_t kPayloadType = 17; + const uint16_t kSequenceNumber = 4711; + const uint32_t kTimestamp = 47114711; + const uint32_t kSsrc = 0x12345678; + MakeRtpHeader(kPayloadType, kSequenceNumber, kTimestamp, kSsrc, + packet_memory.MutableData()); + const double kPacketTime = 1.0; + Packet packet(std::move(packet_memory), kPacketTime); + ASSERT_TRUE(packet.valid_header()); + EXPECT_EQ(kPayloadType, packet.header().payloadType); + EXPECT_EQ(kSequenceNumber, packet.header().sequenceNumber); + EXPECT_EQ(kTimestamp, packet.header().timestamp); + EXPECT_EQ(kSsrc, packet.header().ssrc); + EXPECT_EQ(0, packet.header().numCSRCs); + EXPECT_EQ(kPacketLengthBytes, packet.packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.payload_length_bytes()); + EXPECT_EQ(kPacketLengthBytes, packet.virtual_packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.virtual_payload_length_bytes()); + EXPECT_EQ(kPacketTime, packet.time_ms()); +} + +TEST(TestPacket, DummyPacket) { + const size_t kPacketLengthBytes = kHeaderLengthBytes; // Only RTP header. + const size_t kVirtualPacketLengthBytes = 100; + rtc::CopyOnWriteBuffer packet_memory(kPacketLengthBytes); + const uint8_t kPayloadType = 17; + const uint16_t kSequenceNumber = 4711; + const uint32_t kTimestamp = 47114711; + const uint32_t kSsrc = 0x12345678; + MakeRtpHeader(kPayloadType, kSequenceNumber, kTimestamp, kSsrc, + packet_memory.MutableData()); + const double kPacketTime = 1.0; + Packet packet(std::move(packet_memory), kVirtualPacketLengthBytes, + kPacketTime); + ASSERT_TRUE(packet.valid_header()); + EXPECT_EQ(kPayloadType, packet.header().payloadType); + EXPECT_EQ(kSequenceNumber, packet.header().sequenceNumber); + EXPECT_EQ(kTimestamp, packet.header().timestamp); + EXPECT_EQ(kSsrc, packet.header().ssrc); + EXPECT_EQ(0, packet.header().numCSRCs); + EXPECT_EQ(kPacketLengthBytes, packet.packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.payload_length_bytes()); + EXPECT_EQ(kVirtualPacketLengthBytes, packet.virtual_packet_length_bytes()); + EXPECT_EQ(kVirtualPacketLengthBytes - kHeaderLengthBytes, + packet.virtual_payload_length_bytes()); + EXPECT_EQ(kPacketTime, packet.time_ms()); +} + +TEST(TestPacket, DummyPaddingPacket) { + const size_t kPacketLengthBytes = kHeaderLengthBytes; // Only RTP header. + const size_t kVirtualPacketLengthBytes = 100; + rtc::CopyOnWriteBuffer packet_memory(kPacketLengthBytes); + const uint8_t kPayloadType = 17; + const uint16_t kSequenceNumber = 4711; + const uint32_t kTimestamp = 47114711; + const uint32_t kSsrc = 0x12345678; + MakeRtpHeader(kPayloadType, kSequenceNumber, kTimestamp, kSsrc, + packet_memory.MutableData()); + packet_memory.MutableData()[0] |= 0b0010'0000; // Set the padding bit. + const double kPacketTime = 1.0; + Packet packet(std::move(packet_memory), kVirtualPacketLengthBytes, + kPacketTime); + ASSERT_TRUE(packet.valid_header()); + EXPECT_EQ(kPayloadType, packet.header().payloadType); + EXPECT_EQ(kSequenceNumber, packet.header().sequenceNumber); + EXPECT_EQ(kTimestamp, packet.header().timestamp); + EXPECT_EQ(kSsrc, packet.header().ssrc); + EXPECT_EQ(0, packet.header().numCSRCs); + EXPECT_EQ(kPacketLengthBytes, packet.packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.payload_length_bytes()); + EXPECT_EQ(kVirtualPacketLengthBytes, packet.virtual_packet_length_bytes()); + EXPECT_EQ(kVirtualPacketLengthBytes - kHeaderLengthBytes, + packet.virtual_payload_length_bytes()); + EXPECT_EQ(kPacketTime, packet.time_ms()); +} + +namespace { +// Writes one RED block header starting at `rtp_data`, according to RFC 2198. +// returns the number of bytes written (1 or 4). +// +// Format if `last_payoad` is false: +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |1| block PT | timestamp offset | block length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Format if `last_payoad` is true: +// 0 1 2 3 4 5 6 7 +// +-+-+-+-+-+-+-+-+ +// |0| Block PT | +// +-+-+-+-+-+-+-+-+ + +int MakeRedHeader(int payload_type, + uint32_t timestamp_offset, + int block_length, + bool last_payload, + uint8_t* rtp_data) { + rtp_data[0] = 0x80 | (payload_type & 0x7F); // Set the first bit to 1. + if (last_payload) { + rtp_data[0] &= 0x7F; // Reset the first but to 0 to indicate last block. + return 1; + } + rtp_data[1] = timestamp_offset >> 6; + rtp_data[2] = (timestamp_offset & 0x3F) << 2; + rtp_data[2] |= block_length >> 8; + rtp_data[3] = block_length & 0xFF; + return 4; +} +} // namespace + +TEST(TestPacket, RED) { + const size_t kPacketLengthBytes = 100; + rtc::CopyOnWriteBuffer packet_memory(kPacketLengthBytes); + const uint8_t kRedPayloadType = 17; + const uint16_t kSequenceNumber = 4711; + const uint32_t kTimestamp = 47114711; + const uint32_t kSsrc = 0x12345678; + MakeRtpHeader(kRedPayloadType, kSequenceNumber, kTimestamp, kSsrc, + packet_memory.MutableData()); + // Create four RED headers. + // Payload types are just the same as the block index the offset is 100 times + // the block index. + const int kRedBlocks = 4; + uint8_t* payload_ptr = packet_memory.MutableData() + + kHeaderLengthBytes; // First byte after header. + for (int i = 0; i < kRedBlocks; ++i) { + int payload_type = i; + // Offset value is not used for the last block. + uint32_t timestamp_offset = 100 * i; + int block_length = 10 * i; + bool last_block = (i == kRedBlocks - 1) ? true : false; + payload_ptr += MakeRedHeader(payload_type, timestamp_offset, block_length, + last_block, payload_ptr); + } + const double kPacketTime = 1.0; + // Hand over ownership of `packet_memory` to `packet`. + Packet packet(packet_memory, kPacketLengthBytes, kPacketTime); + ASSERT_TRUE(packet.valid_header()); + EXPECT_EQ(kRedPayloadType, packet.header().payloadType); + EXPECT_EQ(kSequenceNumber, packet.header().sequenceNumber); + EXPECT_EQ(kTimestamp, packet.header().timestamp); + EXPECT_EQ(kSsrc, packet.header().ssrc); + EXPECT_EQ(0, packet.header().numCSRCs); + EXPECT_EQ(kPacketLengthBytes, packet.packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.payload_length_bytes()); + EXPECT_EQ(kPacketLengthBytes, packet.virtual_packet_length_bytes()); + EXPECT_EQ(kPacketLengthBytes - kHeaderLengthBytes, + packet.virtual_payload_length_bytes()); + EXPECT_EQ(kPacketTime, packet.time_ms()); + std::list red_headers; + EXPECT_TRUE(packet.ExtractRedHeaders(&red_headers)); + EXPECT_EQ(kRedBlocks, static_cast(red_headers.size())); + int block_index = 0; + for (std::list::reverse_iterator it = red_headers.rbegin(); + it != red_headers.rend(); ++it) { + // Reading list from the back, since the extraction puts the main payload + // (which is the last one on wire) first. + RTPHeader* red_block = *it; + EXPECT_EQ(block_index, red_block->payloadType); + EXPECT_EQ(kSequenceNumber, red_block->sequenceNumber); + if (block_index == kRedBlocks - 1) { + // Last block has zero offset per definition. + EXPECT_EQ(kTimestamp, red_block->timestamp); + } else { + EXPECT_EQ(kTimestamp - 100 * block_index, red_block->timestamp); + } + EXPECT_EQ(kSsrc, red_block->ssrc); + EXPECT_EQ(0, red_block->numCSRCs); + ++block_index; + } + Packet::DeleteRedHeaders(&red_headers); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.cc new file mode 100644 index 0000000000..5050e1fb17 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.cc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/resample_input_audio_file.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +bool ResampleInputAudioFile::Read(size_t samples, + int output_rate_hz, + int16_t* destination) { + const size_t samples_to_read = samples * file_rate_hz_ / output_rate_hz; + RTC_CHECK_EQ(samples_to_read * output_rate_hz, samples * file_rate_hz_) + << "Frame size and sample rates don't add up to an integer."; + std::unique_ptr temp_destination(new int16_t[samples_to_read]); + if (!InputAudioFile::Read(samples_to_read, temp_destination.get())) + return false; + resampler_.ResetIfNeeded(file_rate_hz_, output_rate_hz, 1); + size_t output_length = 0; + RTC_CHECK_EQ(resampler_.Push(temp_destination.get(), samples_to_read, + destination, samples, output_length), + 0); + RTC_CHECK_EQ(samples, output_length); + return true; +} + +bool ResampleInputAudioFile::Read(size_t samples, int16_t* destination) { + RTC_CHECK_GT(output_rate_hz_, 0) << "Output rate not set."; + return Read(samples, output_rate_hz_, destination); +} + +void ResampleInputAudioFile::set_output_rate_hz(int rate_hz) { + output_rate_hz_ = rate_hz; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h new file mode 100644 index 0000000000..279fece616 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_RESAMPLE_INPUT_AUDIO_FILE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_RESAMPLE_INPUT_AUDIO_FILE_H_ + +#include + +#include "absl/strings/string_view.h" +#include "common_audio/resampler/include/resampler.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" + +namespace webrtc { +namespace test { + +// Class for handling a looping input audio file with resampling. +class ResampleInputAudioFile : public InputAudioFile { + public: + ResampleInputAudioFile(absl::string_view file_name, + int file_rate_hz, + bool loop_at_end = true) + : InputAudioFile(file_name, loop_at_end), + file_rate_hz_(file_rate_hz), + output_rate_hz_(-1) {} + ResampleInputAudioFile(absl::string_view file_name, + int file_rate_hz, + int output_rate_hz, + bool loop_at_end = true) + : InputAudioFile(file_name, loop_at_end), + file_rate_hz_(file_rate_hz), + output_rate_hz_(output_rate_hz) {} + + ResampleInputAudioFile(const ResampleInputAudioFile&) = delete; + ResampleInputAudioFile& operator=(const ResampleInputAudioFile&) = delete; + + bool Read(size_t samples, int output_rate_hz, int16_t* destination); + bool Read(size_t samples, int16_t* destination) override; + void set_output_rate_hz(int rate_hz); + + private: + const int file_rate_hz_; + int output_rate_hz_; + Resampler resampler_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_RESAMPLE_INPUT_AUDIO_FILE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.cc new file mode 100644 index 0000000000..1407aab5f2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/rtc_event_log_source.h" + +#include + +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "logging/rtc_event_log/rtc_event_processor.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +namespace { +bool ShouldSkipStream(ParsedRtcEventLog::MediaType media_type, + uint32_t ssrc, + absl::optional ssrc_filter) { + if (media_type != ParsedRtcEventLog::MediaType::AUDIO) + return true; + if (ssrc_filter.has_value() && ssrc != *ssrc_filter) + return true; + return false; +} +} // namespace + +std::unique_ptr RtcEventLogSource::CreateFromFile( + absl::string_view file_name, + absl::optional ssrc_filter) { + auto source = std::unique_ptr(new RtcEventLogSource()); + ParsedRtcEventLog parsed_log; + auto status = parsed_log.ParseFile(file_name); + if (!status.ok()) { + std::cerr << "Failed to parse event log: " << status.message() << std::endl; + std::cerr << "Skipping log." << std::endl; + return nullptr; + } + if (!source->Initialize(parsed_log, ssrc_filter)) { + std::cerr << "Failed to initialize source from event log, skipping." + << std::endl; + return nullptr; + } + return source; +} + +std::unique_ptr RtcEventLogSource::CreateFromString( + absl::string_view file_contents, + absl::optional ssrc_filter) { + auto source = std::unique_ptr(new RtcEventLogSource()); + ParsedRtcEventLog parsed_log; + auto status = parsed_log.ParseString(file_contents); + if (!status.ok()) { + std::cerr << "Failed to parse event log: " << status.message() << std::endl; + std::cerr << "Skipping log." << std::endl; + return nullptr; + } + if (!source->Initialize(parsed_log, ssrc_filter)) { + std::cerr << "Failed to initialize source from event log, skipping." + << std::endl; + return nullptr; + } + return source; +} + +RtcEventLogSource::~RtcEventLogSource() {} + +std::unique_ptr RtcEventLogSource::NextPacket() { + if (rtp_packet_index_ >= rtp_packets_.size()) + return nullptr; + + std::unique_ptr packet = std::move(rtp_packets_[rtp_packet_index_++]); + return packet; +} + +int64_t RtcEventLogSource::NextAudioOutputEventMs() { + if (audio_output_index_ >= audio_outputs_.size()) + return std::numeric_limits::max(); + + int64_t output_time_ms = audio_outputs_[audio_output_index_++]; + return output_time_ms; +} + +RtcEventLogSource::RtcEventLogSource() : PacketSource() {} + +bool RtcEventLogSource::Initialize(const ParsedRtcEventLog& parsed_log, + absl::optional ssrc_filter) { + const auto first_log_end_time_us = + parsed_log.stop_log_events().empty() + ? std::numeric_limits::max() + : parsed_log.stop_log_events().front().log_time_us(); + + std::set packet_ssrcs; + auto handle_rtp_packet = + [this, first_log_end_time_us, + &packet_ssrcs](const webrtc::LoggedRtpPacketIncoming& incoming) { + if (!filter_.test(incoming.rtp.header.payloadType) && + incoming.log_time_us() < first_log_end_time_us) { + rtp_packets_.emplace_back(std::make_unique( + incoming.rtp.header, incoming.rtp.total_length, + incoming.rtp.total_length - incoming.rtp.header_length, + static_cast(incoming.log_time_ms()))); + packet_ssrcs.insert(rtp_packets_.back()->header().ssrc); + } + }; + + std::set ignored_ssrcs; + auto handle_audio_playout = + [this, first_log_end_time_us, &packet_ssrcs, + &ignored_ssrcs](const webrtc::LoggedAudioPlayoutEvent& audio_playout) { + if (audio_playout.log_time_us() < first_log_end_time_us) { + if (packet_ssrcs.count(audio_playout.ssrc) > 0) { + audio_outputs_.emplace_back(audio_playout.log_time_ms()); + } else { + ignored_ssrcs.insert(audio_playout.ssrc); + } + } + }; + + // This wouldn't be needed if we knew that there was at most one audio stream. + webrtc::RtcEventProcessor event_processor; + for (const auto& rtp_packets : parsed_log.incoming_rtp_packets_by_ssrc()) { + ParsedRtcEventLog::MediaType media_type = + parsed_log.GetMediaType(rtp_packets.ssrc, webrtc::kIncomingPacket); + if (ShouldSkipStream(media_type, rtp_packets.ssrc, ssrc_filter)) { + continue; + } + event_processor.AddEvents(rtp_packets.incoming_packets, handle_rtp_packet); + // If no SSRC filter has been set, use the first SSRC only. The simulator + // does not work properly with interleaved packets from multiple SSRCs. + if (!ssrc_filter.has_value()) { + ssrc_filter = rtp_packets.ssrc; + } + } + + for (const auto& audio_playouts : parsed_log.audio_playout_events()) { + if (ssrc_filter.has_value() && audio_playouts.first != *ssrc_filter) + continue; + event_processor.AddEvents(audio_playouts.second, handle_audio_playout); + } + + // Fills in rtp_packets_ and audio_outputs_. + event_processor.ProcessEventsInOrder(); + + for (const auto& ssrc : ignored_ssrcs) { + std::cout << "Ignoring GetAudio events from SSRC 0x" << std::hex << ssrc + << " because no packets were found with a matching SSRC." + << std::endl; + } + + return true; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.h new file mode 100644 index 0000000000..c67912a67d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtc_event_log_source.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_RTC_EVENT_LOG_SOURCE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_RTC_EVENT_LOG_SOURCE_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "logging/rtc_event_log/rtc_event_log_parser.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { + +class RtpHeaderParser; + +namespace test { + +class Packet; + +class RtcEventLogSource : public PacketSource { + public: + // Creates an RtcEventLogSource reading from `file_name`. If the file cannot + // be opened, or has the wrong format, NULL will be returned. + static std::unique_ptr CreateFromFile( + absl::string_view file_name, + absl::optional ssrc_filter); + // Same as above, but uses a string with the file contents. + static std::unique_ptr CreateFromString( + absl::string_view file_contents, + absl::optional ssrc_filter); + + virtual ~RtcEventLogSource(); + + RtcEventLogSource(const RtcEventLogSource&) = delete; + RtcEventLogSource& operator=(const RtcEventLogSource&) = delete; + + std::unique_ptr NextPacket() override; + + // Returns the timestamp of the next audio output event, in milliseconds. The + // maximum value of int64_t is returned if there are no more audio output + // events available. + int64_t NextAudioOutputEventMs(); + + private: + RtcEventLogSource(); + + bool Initialize(const ParsedRtcEventLog& parsed_log, + absl::optional ssrc_filter); + + std::vector> rtp_packets_; + size_t rtp_packet_index_ = 0; + std::vector audio_outputs_; + size_t audio_output_index_ = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_RTC_EVENT_LOG_SOURCE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_analyze.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_analyze.cc new file mode 100644 index 0000000000..7ecf925ebb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_analyze.cc @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "modules/audio_coding/neteq/tools/packet.h" +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" + +ABSL_FLAG(int, red, 117, "RTP payload type for RED"); +ABSL_FLAG(int, + audio_level, + -1, + "Extension ID for audio level (RFC 6464); " + "-1 not to print audio level"); +ABSL_FLAG(int, + abs_send_time, + -1, + "Extension ID for absolute sender time; " + "-1 not to print absolute send time"); + +int main(int argc, char* argv[]) { + std::vector args = absl::ParseCommandLine(argc, argv); + std::string usage = + "Tool for parsing an RTP dump file to text output.\n" + "Example usage:\n" + "./rtp_analyze input.rtp output.txt\n\n" + "Output is sent to stdout if no output file is given. " + "Note that this tool can read files with or without payloads.\n"; + if (args.size() != 2 && args.size() != 3) { + printf("%s", usage.c_str()); + return 1; + } + + RTC_CHECK(absl::GetFlag(FLAGS_red) >= 0 && + absl::GetFlag(FLAGS_red) <= 127); // Payload type + RTC_CHECK(absl::GetFlag(FLAGS_audio_level) == -1 || // Default + (absl::GetFlag(FLAGS_audio_level) > 0 && + absl::GetFlag(FLAGS_audio_level) <= 255)); // Extension ID + RTC_CHECK(absl::GetFlag(FLAGS_abs_send_time) == -1 || // Default + (absl::GetFlag(FLAGS_abs_send_time) > 0 && + absl::GetFlag(FLAGS_abs_send_time) <= 255)); // Extension ID + + printf("Input file: %s\n", args[1]); + std::unique_ptr file_source( + webrtc::test::RtpFileSource::Create(args[1])); + RTC_DCHECK(file_source.get()); + // Set RTP extension IDs. + bool print_audio_level = false; + if (absl::GetFlag(FLAGS_audio_level) != -1) { + print_audio_level = true; + file_source->RegisterRtpHeaderExtension(webrtc::kRtpExtensionAudioLevel, + absl::GetFlag(FLAGS_audio_level)); + } + bool print_abs_send_time = false; + if (absl::GetFlag(FLAGS_abs_send_time) != -1) { + print_abs_send_time = true; + file_source->RegisterRtpHeaderExtension( + webrtc::kRtpExtensionAbsoluteSendTime, + absl::GetFlag(FLAGS_abs_send_time)); + } + + FILE* out_file; + if (args.size() == 3) { + out_file = fopen(args[2], "wt"); + if (!out_file) { + printf("Cannot open output file %s\n", args[2]); + return -1; + } + printf("Output file: %s\n\n", args[2]); + } else { + out_file = stdout; + } + + // Print file header. + fprintf(out_file, "SeqNo TimeStamp SendTime Size PT M SSRC"); + if (print_audio_level) { + fprintf(out_file, " AuLvl (V)"); + } + if (print_abs_send_time) { + fprintf(out_file, " AbsSendTime"); + } + fprintf(out_file, "\n"); + + uint32_t max_abs_send_time = 0; + int cycles = -1; + std::unique_ptr packet; + while (true) { + packet = file_source->NextPacket(); + if (!packet.get()) { + // End of file reached. + break; + } + // Write packet data to file. Use virtual_packet_length_bytes so that the + // correct packet sizes are printed also for RTP header-only dumps. + fprintf(out_file, "%5u %10u %10u %5i %5i %2i %#08X", + packet->header().sequenceNumber, packet->header().timestamp, + static_cast(packet->time_ms()), + static_cast(packet->virtual_packet_length_bytes()), + packet->header().payloadType, packet->header().markerBit, + packet->header().ssrc); + if (print_audio_level && packet->header().extension.hasAudioLevel) { + fprintf(out_file, " %5u (%1i)", packet->header().extension.audioLevel, + packet->header().extension.voiceActivity); + } + if (print_abs_send_time && packet->header().extension.hasAbsoluteSendTime) { + if (cycles == -1) { + // Initialize. + max_abs_send_time = packet->header().extension.absoluteSendTime; + cycles = 0; + } + // Abs sender time is 24 bit 6.18 fixed point. Shift by 8 to normalize to + // 32 bits (unsigned). Calculate the difference between this packet's + // send time and the maximum observed. Cast to signed 32-bit to get the + // desired wrap-around behavior. + if (static_cast( + (packet->header().extension.absoluteSendTime << 8) - + (max_abs_send_time << 8)) >= 0) { + // The difference is non-negative, meaning that this packet is newer + // than the previously observed maximum absolute send time. + if (packet->header().extension.absoluteSendTime < max_abs_send_time) { + // Wrap detected. + cycles++; + } + max_abs_send_time = packet->header().extension.absoluteSendTime; + } + // Abs sender time is 24 bit 6.18 fixed point. Divide by 2^18 to convert + // to floating point representation. + double send_time_seconds = + static_cast(packet->header().extension.absoluteSendTime) / + 262144 + + 64.0 * cycles; + fprintf(out_file, " %11f", send_time_seconds); + } + fprintf(out_file, "\n"); + + if (packet->header().payloadType == absl::GetFlag(FLAGS_red)) { + std::list red_headers; + packet->ExtractRedHeaders(&red_headers); + while (!red_headers.empty()) { + webrtc::RTPHeader* red = red_headers.front(); + RTC_DCHECK(red); + fprintf(out_file, "* %5u %10u %10u %5i\n", red->sequenceNumber, + red->timestamp, static_cast(packet->time_ms()), + red->payloadType); + red_headers.pop_front(); + delete red; + } + } + } + + fclose(out_file); + + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_encode.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_encode.cc new file mode 100644 index 0000000000..8adca927f0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_encode.cc @@ -0,0 +1,351 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#ifdef WIN32 +#include +#endif +#if defined(WEBRTC_LINUX) || defined(WEBRTC_FUCHSIA) +#include +#endif + +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/memory/memory.h" +#include "api/audio/audio_frame.h" +#include "api/audio_codecs/L16/audio_encoder_L16.h" +#include "api/audio_codecs/g711/audio_encoder_g711.h" +#include "api/audio_codecs/g722/audio_encoder_g722.h" +#include "api/audio_codecs/ilbc/audio_encoder_ilbc.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/neteq/tools/input_audio_file.h" +#include "rtc_base/numerics/safe_conversions.h" + +ABSL_FLAG(bool, list_codecs, false, "Enumerate all codecs"); +ABSL_FLAG(std::string, codec, "opus", "Codec to use"); +ABSL_FLAG(int, + frame_len, + 0, + "Frame length in ms; 0 indicates codec default value"); +ABSL_FLAG(int, bitrate, 0, "Bitrate in kbps; 0 indicates codec default value"); +ABSL_FLAG(int, + payload_type, + -1, + "RTP payload type; -1 indicates codec default value"); +ABSL_FLAG(int, + cng_payload_type, + -1, + "RTP payload type for CNG; -1 indicates default value"); +ABSL_FLAG(int, ssrc, 0, "SSRC to write to the RTP header"); +ABSL_FLAG(bool, dtx, false, "Use DTX/CNG"); +ABSL_FLAG(int, sample_rate, 48000, "Sample rate of the input file"); + +namespace webrtc { +namespace test { +namespace { + +// Add new codecs here, and to the map below. +enum class CodecType { + kOpus, + kPcmU, + kPcmA, + kG722, + kPcm16b8, + kPcm16b16, + kPcm16b32, + kPcm16b48, + kIlbc, +}; + +struct CodecTypeAndInfo { + CodecType type; + int default_payload_type; + bool internal_dtx; +}; + +// List all supported codecs here. This map defines the command-line parameter +// value (the key string) for selecting each codec, together with information +// whether it is using internal or external DTX/CNG. +const std::map& CodecList() { + static const auto* const codec_list = + new std::map{ + {"opus", {CodecType::kOpus, 111, true}}, + {"pcmu", {CodecType::kPcmU, 0, false}}, + {"pcma", {CodecType::kPcmA, 8, false}}, + {"g722", {CodecType::kG722, 9, false}}, + {"pcm16b_8", {CodecType::kPcm16b8, 93, false}}, + {"pcm16b_16", {CodecType::kPcm16b16, 94, false}}, + {"pcm16b_32", {CodecType::kPcm16b32, 95, false}}, + {"pcm16b_48", {CodecType::kPcm16b48, 96, false}}, + {"ilbc", {CodecType::kIlbc, 102, false}}}; + return *codec_list; +} + +// This class will receive callbacks from ACM when a packet is ready, and write +// it to the output file. +class Packetizer : public AudioPacketizationCallback { + public: + Packetizer(FILE* out_file, uint32_t ssrc, int timestamp_rate_hz) + : out_file_(out_file), + ssrc_(ssrc), + timestamp_rate_hz_(timestamp_rate_hz) {} + + int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + int64_t absolute_capture_timestamp_ms) override { + if (payload_len_bytes == 0) { + return 0; + } + + constexpr size_t kRtpHeaderLength = 12; + constexpr size_t kRtpDumpHeaderLength = 8; + const uint16_t length = htons(rtc::checked_cast( + kRtpHeaderLength + kRtpDumpHeaderLength + payload_len_bytes)); + const uint16_t plen = htons( + rtc::checked_cast(kRtpHeaderLength + payload_len_bytes)); + const uint32_t offset = htonl(timestamp / (timestamp_rate_hz_ / 1000)); + RTC_CHECK_EQ(fwrite(&length, sizeof(uint16_t), 1, out_file_), 1); + RTC_CHECK_EQ(fwrite(&plen, sizeof(uint16_t), 1, out_file_), 1); + RTC_CHECK_EQ(fwrite(&offset, sizeof(uint32_t), 1, out_file_), 1); + + const uint8_t rtp_header[] = {0x80, + static_cast(payload_type & 0x7F), + static_cast(sequence_number_ >> 8), + static_cast(sequence_number_), + static_cast(timestamp >> 24), + static_cast(timestamp >> 16), + static_cast(timestamp >> 8), + static_cast(timestamp), + static_cast(ssrc_ >> 24), + static_cast(ssrc_ >> 16), + static_cast(ssrc_ >> 8), + static_cast(ssrc_)}; + static_assert(sizeof(rtp_header) == kRtpHeaderLength, ""); + RTC_CHECK_EQ( + fwrite(rtp_header, sizeof(uint8_t), kRtpHeaderLength, out_file_), + kRtpHeaderLength); + ++sequence_number_; // Intended to wrap on overflow. + + RTC_CHECK_EQ( + fwrite(payload_data, sizeof(uint8_t), payload_len_bytes, out_file_), + payload_len_bytes); + + return 0; + } + + private: + FILE* const out_file_; + const uint32_t ssrc_; + const int timestamp_rate_hz_; + uint16_t sequence_number_ = 0; +}; + +void SetFrameLenIfFlagIsPositive(int* config_frame_len) { + if (absl::GetFlag(FLAGS_frame_len) > 0) { + *config_frame_len = absl::GetFlag(FLAGS_frame_len); + } +} + +template +typename T::Config GetCodecConfig() { + typename T::Config config; + SetFrameLenIfFlagIsPositive(&config.frame_size_ms); + RTC_CHECK(config.IsOk()); + return config; +} + +AudioEncoderL16::Config Pcm16bConfig(CodecType codec_type) { + auto config = GetCodecConfig(); + switch (codec_type) { + case CodecType::kPcm16b8: + config.sample_rate_hz = 8000; + return config; + case CodecType::kPcm16b16: + config.sample_rate_hz = 16000; + return config; + case CodecType::kPcm16b32: + config.sample_rate_hz = 32000; + return config; + case CodecType::kPcm16b48: + config.sample_rate_hz = 48000; + return config; + default: + RTC_DCHECK_NOTREACHED(); + return config; + } +} + +std::unique_ptr CreateEncoder(CodecType codec_type, + int payload_type) { + switch (codec_type) { + case CodecType::kOpus: { + AudioEncoderOpus::Config config = GetCodecConfig(); + if (absl::GetFlag(FLAGS_bitrate) > 0) { + config.bitrate_bps = absl::GetFlag(FLAGS_bitrate); + } + config.dtx_enabled = absl::GetFlag(FLAGS_dtx); + RTC_CHECK(config.IsOk()); + return AudioEncoderOpus::MakeAudioEncoder(config, payload_type); + } + + case CodecType::kPcmU: + case CodecType::kPcmA: { + AudioEncoderG711::Config config = GetCodecConfig(); + config.type = codec_type == CodecType::kPcmU + ? AudioEncoderG711::Config::Type::kPcmU + : AudioEncoderG711::Config::Type::kPcmA; + RTC_CHECK(config.IsOk()); + return AudioEncoderG711::MakeAudioEncoder(config, payload_type); + } + + case CodecType::kG722: { + return AudioEncoderG722::MakeAudioEncoder( + GetCodecConfig(), payload_type); + } + + case CodecType::kPcm16b8: + case CodecType::kPcm16b16: + case CodecType::kPcm16b32: + case CodecType::kPcm16b48: { + return AudioEncoderL16::MakeAudioEncoder(Pcm16bConfig(codec_type), + payload_type); + } + + case CodecType::kIlbc: { + return AudioEncoderIlbc::MakeAudioEncoder( + GetCodecConfig(), payload_type); + } + } + RTC_DCHECK_NOTREACHED(); + return nullptr; +} + +AudioEncoderCngConfig GetCngConfig(int sample_rate_hz) { + AudioEncoderCngConfig cng_config; + const auto default_payload_type = [&] { + switch (sample_rate_hz) { + case 8000: + return 13; + case 16000: + return 98; + case 32000: + return 99; + case 48000: + return 100; + default: + RTC_DCHECK_NOTREACHED(); + } + return 0; + }; + cng_config.payload_type = absl::GetFlag(FLAGS_cng_payload_type) != -1 + ? absl::GetFlag(FLAGS_cng_payload_type) + : default_payload_type(); + return cng_config; +} + +int RunRtpEncode(int argc, char* argv[]) { + std::vector args = absl::ParseCommandLine(argc, argv); + const std::string usage = + "Tool for generating an RTP dump file from audio input.\n" + "Example usage:\n" + "./rtp_encode input.pcm output.rtp --codec=[codec] " + "--frame_len=[frame_len] --bitrate=[bitrate]\n\n"; + if (!absl::GetFlag(FLAGS_list_codecs) && args.size() != 3) { + printf("%s", usage.c_str()); + return 1; + } + + if (absl::GetFlag(FLAGS_list_codecs)) { + printf("The following arguments are valid --codec parameters:\n"); + for (const auto& c : CodecList()) { + printf(" %s\n", c.first.c_str()); + } + return 0; + } + + const auto codec_it = CodecList().find(absl::GetFlag(FLAGS_codec)); + if (codec_it == CodecList().end()) { + printf("%s is not a valid codec name.\n", + absl::GetFlag(FLAGS_codec).c_str()); + printf("Use argument --list_codecs to see all valid codec names.\n"); + return 1; + } + + // Create the codec. + const int payload_type = absl::GetFlag(FLAGS_payload_type) == -1 + ? codec_it->second.default_payload_type + : absl::GetFlag(FLAGS_payload_type); + std::unique_ptr codec = + CreateEncoder(codec_it->second.type, payload_type); + + // Create an external VAD/CNG encoder if needed. + if (absl::GetFlag(FLAGS_dtx) && !codec_it->second.internal_dtx) { + AudioEncoderCngConfig cng_config = GetCngConfig(codec->SampleRateHz()); + RTC_DCHECK(codec); + cng_config.speech_encoder = std::move(codec); + codec = CreateComfortNoiseEncoder(std::move(cng_config)); + } + RTC_DCHECK(codec); + + // Set up ACM. + const int timestamp_rate_hz = codec->RtpTimestampRateHz(); + AudioCodingModule::Config config; + std::unique_ptr acm(AudioCodingModule::Create(config)); + acm->SetEncoder(std::move(codec)); + + // Open files. + printf("Input file: %s\n", args[1]); + InputAudioFile input_file(args[1], false); // Open input in non-looping mode. + FILE* out_file = fopen(args[2], "wb"); + RTC_CHECK(out_file) << "Could not open file " << args[2] << " for writing"; + printf("Output file: %s\n", args[2]); + fprintf(out_file, "#!rtpplay1.0 \n"); //, + // Write 3 32-bit values followed by 2 16-bit values, all set to 0. This means + // a total of 16 bytes. + const uint8_t file_header[16] = {0}; + RTC_CHECK_EQ(fwrite(file_header, sizeof(file_header), 1, out_file), 1); + + // Create and register the packetizer, which will write the packets to file. + Packetizer packetizer(out_file, absl::GetFlag(FLAGS_ssrc), timestamp_rate_hz); + RTC_DCHECK_EQ(acm->RegisterTransportCallback(&packetizer), 0); + + AudioFrame audio_frame; + audio_frame.samples_per_channel_ = + absl::GetFlag(FLAGS_sample_rate) / 100; // 10 ms + audio_frame.sample_rate_hz_ = absl::GetFlag(FLAGS_sample_rate); + audio_frame.num_channels_ = 1; + + while (input_file.Read(audio_frame.samples_per_channel_, + audio_frame.mutable_data())) { + RTC_CHECK_GE(acm->Add10MsData(audio_frame), 0); + audio_frame.timestamp_ += audio_frame.samples_per_channel_; + } + + return 0; +} + +} // namespace +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::RunRtpEncode(argc, argv); +} diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_file_source.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_file_source.cc new file mode 100644 index 0000000000..a43c29638c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_file_source.cc @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/rtp_file_source.h" + +#include + +#include "absl/strings/string_view.h" +#ifndef WIN32 +#include +#endif + +#include + +#include "modules/audio_coding/neteq/tools/packet.h" +#include "rtc_base/checks.h" +#include "test/rtp_file_reader.h" + +namespace webrtc { +namespace test { + +RtpFileSource* RtpFileSource::Create(absl::string_view file_name, + absl::optional ssrc_filter) { + RtpFileSource* source = new RtpFileSource(ssrc_filter); + RTC_CHECK(source->OpenFile(file_name)); + return source; +} + +bool RtpFileSource::ValidRtpDump(absl::string_view file_name) { + std::unique_ptr temp_file( + RtpFileReader::Create(RtpFileReader::kRtpDump, file_name)); + return !!temp_file; +} + +bool RtpFileSource::ValidPcap(absl::string_view file_name) { + std::unique_ptr temp_file( + RtpFileReader::Create(RtpFileReader::kPcap, file_name)); + return !!temp_file; +} + +RtpFileSource::~RtpFileSource() {} + +bool RtpFileSource::RegisterRtpHeaderExtension(RTPExtensionType type, + uint8_t id) { + return rtp_header_extension_map_.RegisterByType(id, type); +} + +std::unique_ptr RtpFileSource::NextPacket() { + while (true) { + RtpPacket temp_packet; + if (!rtp_reader_->NextPacket(&temp_packet)) { + return NULL; + } + if (temp_packet.original_length == 0) { + // May be an RTCP packet. + // Read the next one. + continue; + } + auto packet = std::make_unique( + rtc::CopyOnWriteBuffer(temp_packet.data, temp_packet.length), + temp_packet.original_length, temp_packet.time_ms, + &rtp_header_extension_map_); + if (!packet->valid_header()) { + continue; + } + if (filter_.test(packet->header().payloadType) || + (ssrc_filter_ && packet->header().ssrc != *ssrc_filter_)) { + // This payload type should be filtered out. Continue to the next packet. + continue; + } + return packet; + } +} + +RtpFileSource::RtpFileSource(absl::optional ssrc_filter) + : PacketSource(), + ssrc_filter_(ssrc_filter) {} + +bool RtpFileSource::OpenFile(absl::string_view file_name) { + rtp_reader_.reset(RtpFileReader::Create(RtpFileReader::kRtpDump, file_name)); + if (rtp_reader_) + return true; + rtp_reader_.reset(RtpFileReader::Create(RtpFileReader::kPcap, file_name)); + if (!rtp_reader_) { + RTC_FATAL() + << "Couldn't open input file as either a rtpdump or .pcap. Note " + << "that .pcapng is not supported."; + } + return true; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_file_source.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_file_source.h new file mode 100644 index 0000000000..55505be630 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_file_source.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_FILE_SOURCE_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_FILE_SOURCE_H_ + +#include + +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/tools/packet_source.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +namespace webrtc { + +namespace test { + +class RtpFileReader; + +class RtpFileSource : public PacketSource { + public: + // Creates an RtpFileSource reading from `file_name`. If the file cannot be + // opened, or has the wrong format, NULL will be returned. + static RtpFileSource* Create( + absl::string_view file_name, + absl::optional ssrc_filter = absl::nullopt); + + // Checks whether a files is a valid RTP dump or PCAP (Wireshark) file. + static bool ValidRtpDump(absl::string_view file_name); + static bool ValidPcap(absl::string_view file_name); + + ~RtpFileSource() override; + + RtpFileSource(const RtpFileSource&) = delete; + RtpFileSource& operator=(const RtpFileSource&) = delete; + + // Registers an RTP header extension and binds it to `id`. + virtual bool RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id); + + std::unique_ptr NextPacket() override; + + private: + static const int kFirstLineLength = 40; + static const int kRtpFileHeaderSize = 4 + 4 + 4 + 2 + 2; + static const size_t kPacketHeaderSize = 8; + + explicit RtpFileSource(absl::optional ssrc_filter); + + bool OpenFile(absl::string_view file_name); + + std::unique_ptr rtp_reader_; + const absl::optional ssrc_filter_; + RtpHeaderExtensionMap rtp_header_extension_map_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_FILE_SOURCE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_generator.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_generator.cc new file mode 100644 index 0000000000..e883fc11d6 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_generator.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/tools/rtp_generator.h" + + +namespace webrtc { +namespace test { + +uint32_t RtpGenerator::GetRtpHeader(uint8_t payload_type, + size_t payload_length_samples, + RTPHeader* rtp_header) { + RTC_DCHECK(rtp_header); + if (!rtp_header) { + return 0; + } + rtp_header->sequenceNumber = seq_number_++; + rtp_header->timestamp = timestamp_; + timestamp_ += static_cast(payload_length_samples); + rtp_header->payloadType = payload_type; + rtp_header->markerBit = false; + rtp_header->ssrc = ssrc_; + rtp_header->numCSRCs = 0; + + uint32_t this_send_time = next_send_time_ms_; + RTC_DCHECK_GT(samples_per_ms_, 0); + next_send_time_ms_ += + ((1.0 + drift_factor_) * payload_length_samples) / samples_per_ms_; + return this_send_time; +} + +void RtpGenerator::set_drift_factor(double factor) { + if (factor > -1.0) { + drift_factor_ = factor; + } +} + +uint32_t TimestampJumpRtpGenerator::GetRtpHeader(uint8_t payload_type, + size_t payload_length_samples, + RTPHeader* rtp_header) { + uint32_t ret = RtpGenerator::GetRtpHeader(payload_type, + payload_length_samples, rtp_header); + if (timestamp_ - static_cast(payload_length_samples) <= + jump_from_timestamp_ && + timestamp_ > jump_from_timestamp_) { + // We just moved across the `jump_from_timestamp_` timestamp. Do the jump. + timestamp_ = jump_to_timestamp_; + } + return ret; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_generator.h b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_generator.h new file mode 100644 index 0000000000..2e615adec5 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_generator.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_GENERATOR_H_ +#define MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_GENERATOR_H_ + +#include "api/rtp_headers.h" + +namespace webrtc { +namespace test { + +// Class for generating RTP headers. +class RtpGenerator { + public: + RtpGenerator(int samples_per_ms, + uint16_t start_seq_number = 0, + uint32_t start_timestamp = 0, + uint32_t start_send_time_ms = 0, + uint32_t ssrc = 0x12345678) + : seq_number_(start_seq_number), + timestamp_(start_timestamp), + next_send_time_ms_(start_send_time_ms), + ssrc_(ssrc), + samples_per_ms_(samples_per_ms), + drift_factor_(0.0) {} + + virtual ~RtpGenerator() {} + + RtpGenerator(const RtpGenerator&) = delete; + RtpGenerator& operator=(const RtpGenerator&) = delete; + + // Writes the next RTP header to `rtp_header`, which will be of type + // `payload_type`. Returns the send time for this packet (in ms). The value of + // `payload_length_samples` determines the send time for the next packet. + virtual uint32_t GetRtpHeader(uint8_t payload_type, + size_t payload_length_samples, + RTPHeader* rtp_header); + + void set_drift_factor(double factor); + + protected: + uint16_t seq_number_; + uint32_t timestamp_; + uint32_t next_send_time_ms_; + const uint32_t ssrc_; + const int samples_per_ms_; + double drift_factor_; +}; + +class TimestampJumpRtpGenerator : public RtpGenerator { + public: + TimestampJumpRtpGenerator(int samples_per_ms, + uint16_t start_seq_number, + uint32_t start_timestamp, + uint32_t jump_from_timestamp, + uint32_t jump_to_timestamp) + : RtpGenerator(samples_per_ms, start_seq_number, start_timestamp), + jump_from_timestamp_(jump_from_timestamp), + jump_to_timestamp_(jump_to_timestamp) {} + + TimestampJumpRtpGenerator(const TimestampJumpRtpGenerator&) = delete; + TimestampJumpRtpGenerator& operator=(const TimestampJumpRtpGenerator&) = + delete; + + uint32_t GetRtpHeader(uint8_t payload_type, + size_t payload_length_samples, + RTPHeader* rtp_header) override; + + private: + uint32_t jump_from_timestamp_; + uint32_t jump_to_timestamp_; +}; + +} // namespace test +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_RTP_GENERATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_jitter.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_jitter.cc new file mode 100644 index 0000000000..cccaa9a3bb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtp_jitter.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include +#include +#include +#include + +#include "api/array_view.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "rtc_base/buffer.h" + +namespace webrtc { +namespace test { +namespace { + +constexpr size_t kRtpDumpHeaderLength = 8; + +// Returns the next packet or an empty buffer if end of file was encountered. +rtc::Buffer ReadNextPacket(FILE* file) { + // Read the rtpdump header for the next packet. + rtc::Buffer buffer; + buffer.SetData(kRtpDumpHeaderLength, [&](rtc::ArrayView x) { + return fread(x.data(), 1, x.size(), file); + }); + if (buffer.size() != kRtpDumpHeaderLength) { + return rtc::Buffer(); + } + + // Get length field. This is the total length for this packet written to file, + // including the kRtpDumpHeaderLength bytes already read. + const uint16_t len = ByteReader::ReadBigEndian(buffer.data()); + RTC_CHECK_GE(len, kRtpDumpHeaderLength); + + // Read remaining data from file directly into buffer. + buffer.AppendData(len - kRtpDumpHeaderLength, [&](rtc::ArrayView x) { + return fread(x.data(), 1, x.size(), file); + }); + if (buffer.size() != len) { + buffer.Clear(); + } + return buffer; +} + +struct PacketAndTime { + rtc::Buffer packet; + int time; +}; + +void WritePacket(const PacketAndTime& packet, FILE* file) { + // Write the first 4 bytes from the original packet. + const auto* payload_ptr = packet.packet.data(); + RTC_CHECK_EQ(fwrite(payload_ptr, 4, 1, file), 1); + payload_ptr += 4; + + // Convert the new time offset to network endian, and write to file. + uint8_t time[sizeof(uint32_t)]; + ByteWriter::WriteBigEndian(time, packet.time); + RTC_CHECK_EQ(fwrite(time, sizeof(uint32_t), 1, file), 1); + payload_ptr += 4; // Skip the old time in the original payload. + + // Write the remaining part of the payload. + RTC_DCHECK_EQ(payload_ptr - packet.packet.data(), kRtpDumpHeaderLength); + RTC_CHECK_EQ( + fwrite(payload_ptr, packet.packet.size() - kRtpDumpHeaderLength, 1, file), + 1); +} + +int RunRtpJitter(int argc, char* argv[]) { + const std::string program_name = argv[0]; + const std::string usage = + "Tool for alternating the arrival times in an RTP dump file.\n" + "Example usage:\n" + + program_name + " input.rtp arrival_times_ms.txt output.rtp\n\n"; + if (argc != 4) { + printf("%s", usage.c_str()); + return 1; + } + + printf("Input RTP file: %s\n", argv[1]); + FILE* in_file = fopen(argv[1], "rb"); + RTC_CHECK(in_file) << "Could not open file " << argv[1] << " for reading"; + printf("Timing file: %s\n", argv[2]); + std::ifstream timing_file(argv[2]); + printf("Output file: %s\n", argv[3]); + FILE* out_file = fopen(argv[3], "wb"); + RTC_CHECK(out_file) << "Could not open file " << argv[2] << " for writing"; + + // Copy the RTP file header to the output file. + char header_string[30]; + RTC_CHECK(fgets(header_string, 30, in_file)); + fprintf(out_file, "%s", header_string); + uint8_t file_header[16]; + RTC_CHECK_EQ(fread(file_header, sizeof(file_header), 1, in_file), 1); + RTC_CHECK_EQ(fwrite(file_header, sizeof(file_header), 1, out_file), 1); + + // Read all time values from the timing file. Store in a vector. + std::vector new_arrival_times; + int new_time; + while (timing_file >> new_time) { + new_arrival_times.push_back(new_time); + } + + // Read all packets from the input RTP file, but no more than the number of + // new time values. Store RTP packets together with new time values. + auto time_it = new_arrival_times.begin(); + std::vector packets; + while (1) { + auto packet = ReadNextPacket(in_file); + if (packet.empty() || time_it == new_arrival_times.end()) { + break; + } + packets.push_back({std::move(packet), *time_it}); + ++time_it; + } + + // Sort on new time values. + std::sort(packets.begin(), packets.end(), + [](const PacketAndTime& a, const PacketAndTime& b) { + return a.time < b.time; + }); + + // Write packets to output file. + for (const auto& p : packets) { + WritePacket(p, out_file); + } + + fclose(in_file); + fclose(out_file); + return 0; +} + +} // namespace +} // namespace test +} // namespace webrtc + +int main(int argc, char* argv[]) { + return webrtc::test::RunRtpJitter(argc, argv); +} diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtpcat.cc b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtpcat.cc new file mode 100644 index 0000000000..431de553ae --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/tools/rtpcat.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include + +#include "rtc_base/checks.h" +#include "test/rtp_file_reader.h" +#include "test/rtp_file_writer.h" + +using webrtc::test::RtpFileReader; +using webrtc::test::RtpFileWriter; + +int main(int argc, char* argv[]) { + if (argc < 3) { + printf("Concatenates multiple rtpdump files into one.\n"); + printf("Usage: rtpcat in1.rtp int2.rtp [...] out.rtp\n"); + exit(1); + } + + std::unique_ptr output( + RtpFileWriter::Create(RtpFileWriter::kRtpDump, argv[argc - 1])); + RTC_CHECK(output.get() != NULL) << "Cannot open output file."; + printf("Output RTP file: %s\n", argv[argc - 1]); + + for (int i = 1; i < argc - 1; i++) { + std::unique_ptr input( + RtpFileReader::Create(RtpFileReader::kRtpDump, argv[i])); + RTC_CHECK(input.get() != NULL) << "Cannot open input file " << argv[i]; + printf("Input RTP file: %s\n", argv[i]); + + webrtc::test::RtpPacket packet; + while (input->NextPacket(&packet)) + RTC_CHECK(output->WritePacket(&packet)); + } + return 0; +} diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.cc b/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.cc new file mode 100644 index 0000000000..baed812327 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/underrun_optimizer.h" + +#include + +namespace webrtc { + +namespace { + +constexpr int kDelayBuckets = 100; +constexpr int kBucketSizeMs = 20; + +} // namespace + +UnderrunOptimizer::UnderrunOptimizer(const TickTimer* tick_timer, + int histogram_quantile, + int forget_factor, + absl::optional start_forget_weight, + absl::optional resample_interval_ms) + : tick_timer_(tick_timer), + histogram_(kDelayBuckets, forget_factor, start_forget_weight), + histogram_quantile_(histogram_quantile), + resample_interval_ms_(resample_interval_ms) {} + +void UnderrunOptimizer::Update(int relative_delay_ms) { + absl::optional histogram_update; + if (resample_interval_ms_) { + if (!resample_stopwatch_) { + resample_stopwatch_ = tick_timer_->GetNewStopwatch(); + } + if (static_cast(resample_stopwatch_->ElapsedMs()) > + *resample_interval_ms_) { + histogram_update = max_delay_in_interval_ms_; + resample_stopwatch_ = tick_timer_->GetNewStopwatch(); + max_delay_in_interval_ms_ = 0; + } + max_delay_in_interval_ms_ = + std::max(max_delay_in_interval_ms_, relative_delay_ms); + } else { + histogram_update = relative_delay_ms; + } + if (!histogram_update) { + return; + } + + const int index = *histogram_update / kBucketSizeMs; + if (index < histogram_.NumBuckets()) { + // Maximum delay to register is 2000 ms. + histogram_.Add(index); + } + int bucket_index = histogram_.Quantile(histogram_quantile_); + optimal_delay_ms_ = (1 + bucket_index) * kBucketSizeMs; +} + +void UnderrunOptimizer::Reset() { + histogram_.Reset(); + resample_stopwatch_.reset(); + max_delay_in_interval_ms_ = 0; + optimal_delay_ms_.reset(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.h b/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.h new file mode 100644 index 0000000000..b37ce18795 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_UNDERRUN_OPTIMIZER_H_ +#define MODULES_AUDIO_CODING_NETEQ_UNDERRUN_OPTIMIZER_H_ + +#include + +#include "absl/types/optional.h" +#include "api/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/histogram.h" + +namespace webrtc { + +// Estimates probability of buffer underrun due to late packet arrival. +// The optimal delay is decided such that the probability of underrun is lower +// than 1 - `histogram_quantile`. +class UnderrunOptimizer { + public: + UnderrunOptimizer(const TickTimer* tick_timer, + int histogram_quantile, + int forget_factor, + absl::optional start_forget_weight, + absl::optional resample_interval_ms); + + void Update(int relative_delay_ms); + + absl::optional GetOptimalDelayMs() const { return optimal_delay_ms_; } + + void Reset(); + + private: + const TickTimer* tick_timer_; + Histogram histogram_; + const int histogram_quantile_; // In Q30. + const absl::optional resample_interval_ms_; + std::unique_ptr resample_stopwatch_; + int max_delay_in_interval_ms_ = 0; + absl::optional optimal_delay_ms_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_UNDERRUN_OPTIMIZER_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer_unittest.cc b/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer_unittest.cc new file mode 100644 index 0000000000..a86e9cf107 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer_unittest.cc @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/underrun_optimizer.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kDefaultHistogramQuantile = 1020054733; // 0.95 in Q30. +constexpr int kForgetFactor = 32745; // 0.9993 in Q15. + +} // namespace + +TEST(UnderrunOptimizerTest, ResamplePacketDelays) { + TickTimer tick_timer; + constexpr int kResampleIntervalMs = 500; + UnderrunOptimizer underrun_optimizer(&tick_timer, kDefaultHistogramQuantile, + kForgetFactor, absl::nullopt, + kResampleIntervalMs); + + // The histogram should be updated once with the maximum delay observed for + // the following sequence of updates. + for (int i = 0; i < 500; i += 20) { + underrun_optimizer.Update(i); + EXPECT_FALSE(underrun_optimizer.GetOptimalDelayMs()); + } + tick_timer.Increment(kResampleIntervalMs / tick_timer.ms_per_tick() + 1); + underrun_optimizer.Update(0); + EXPECT_EQ(underrun_optimizer.GetOptimalDelayMs(), 500); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/neteq_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/neteq_gn/moz.build new file mode 100644 index 0000000000..823aea6dda --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/neteq_gn/moz.build @@ -0,0 +1,267 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/neteq/audio_vector.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/underrun_optimizer.cc" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/neteq/accelerate.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/audio_multi_vector.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/background_noise.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/buffer_level_filter.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/comfort_noise.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/cross_correlation.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/decision_logic.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/decoder_database.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/delay_manager.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/dsp_helper.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_buffer.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/dtmf_tone_generator.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/expand.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/expand_uma_logger.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/histogram.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/merge.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/nack_tracker.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/normal.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/packet.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/packet_arrival_history.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/packet_buffer.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/post_decode_vad.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/preemptive_expand.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/random_vector.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/red_payload_splitter.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/reorder_optimizer.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/statistics_calculator.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/sync_buffer.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/time_stretch.cc", + "/third_party/libwebrtc/modules/audio_coding/neteq/timestamp_scaler.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("neteq_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/pcm16b_c_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/pcm16b_c_gn/moz.build new file mode 100644 index 0000000000..54863e95ae --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/pcm16b_c_gn/moz.build @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b.c" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("pcm16b_c_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/pcm16b_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/pcm16b_gn/moz.build new file mode 100644 index 0000000000..afee108cb3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/pcm16b_gn/moz.build @@ -0,0 +1,227 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_decoder_pcm16b.cc", + "/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/audio_encoder_pcm16b.cc", + "/third_party/libwebrtc/modules/audio_coding/codecs/pcm16b/pcm16b_common.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("pcm16b_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/red_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/red_gn/moz.build new file mode 100644 index 0000000000..c4fa5c8bc4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/red_gn/moz.build @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("red_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/test/Channel.cc b/third_party/libwebrtc/modules/audio_coding/test/Channel.cc new file mode 100644 index 0000000000..35aa6cb6b4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/Channel.cc @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/Channel.h" + +#include + +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +int32_t Channel::SendData(AudioFrameType frameType, + uint8_t payloadType, + uint32_t timeStamp, + const uint8_t* payloadData, + size_t payloadSize, + int64_t absolute_capture_timestamp_ms) { + RTPHeader rtp_header; + int32_t status; + size_t payloadDataSize = payloadSize; + + rtp_header.markerBit = false; + rtp_header.ssrc = 0; + rtp_header.sequenceNumber = + (external_sequence_number_ < 0) + ? _seqNo++ + : static_cast(external_sequence_number_); + rtp_header.payloadType = payloadType; + rtp_header.timestamp = (external_send_timestamp_ < 0) + ? timeStamp + : static_cast(external_send_timestamp_); + + if (frameType == AudioFrameType::kEmptyFrame) { + // When frame is empty, we should not transmit it. The frame size of the + // next non-empty frame will be based on the previous frame size. + _useLastFrameSize = _lastFrameSizeSample > 0; + return 0; + } + + memcpy(_payloadData, payloadData, payloadDataSize); + if (_isStereo) { + if (_leftChannel) { + _rtp_header = rtp_header; + _leftChannel = false; + } else { + rtp_header = _rtp_header; + _leftChannel = true; + } + } + + _channelCritSect.Lock(); + if (_saveBitStream) { + // fwrite(payloadData, sizeof(uint8_t), payloadSize, _bitStreamFile); + } + + if (!_isStereo) { + CalcStatistics(rtp_header, payloadSize); + } + _useLastFrameSize = false; + _lastInTimestamp = timeStamp; + _totalBytes += payloadDataSize; + _channelCritSect.Unlock(); + + if (_useFECTestWithPacketLoss) { + _packetLoss += 1; + if (_packetLoss == 3) { + _packetLoss = 0; + return 0; + } + } + + if (num_packets_to_drop_ > 0) { + num_packets_to_drop_--; + return 0; + } + + status = + _receiverACM->IncomingPacket(_payloadData, payloadDataSize, rtp_header); + + return status; +} + +// TODO(turajs): rewite this method. +void Channel::CalcStatistics(const RTPHeader& rtp_header, size_t payloadSize) { + int n; + if ((rtp_header.payloadType != _lastPayloadType) && + (_lastPayloadType != -1)) { + // payload-type is changed. + // we have to terminate the calculations on the previous payload type + // we ignore the last packet in that payload type just to make things + // easier. + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + if (_lastPayloadType == _payloadStats[n].payloadType) { + _payloadStats[n].newPacket = true; + break; + } + } + } + _lastPayloadType = rtp_header.payloadType; + + bool newPayload = true; + ACMTestPayloadStats* currentPayloadStr = NULL; + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + if (rtp_header.payloadType == _payloadStats[n].payloadType) { + newPayload = false; + currentPayloadStr = &_payloadStats[n]; + break; + } + } + + if (!newPayload) { + if (!currentPayloadStr->newPacket) { + if (!_useLastFrameSize) { + _lastFrameSizeSample = + (uint32_t)((uint32_t)rtp_header.timestamp - + (uint32_t)currentPayloadStr->lastTimestamp); + } + RTC_DCHECK_GT(_lastFrameSizeSample, 0); + int k = 0; + for (; k < MAX_NUM_FRAMESIZES; ++k) { + if ((currentPayloadStr->frameSizeStats[k].frameSizeSample == + _lastFrameSizeSample) || + (currentPayloadStr->frameSizeStats[k].frameSizeSample == 0)) { + break; + } + } + if (k == MAX_NUM_FRAMESIZES) { + // New frame size found but no space to count statistics on it. Skip it. + printf("No memory to store statistics for payload %d : frame size %d\n", + _lastPayloadType, _lastFrameSizeSample); + return; + } + ACMTestFrameSizeStats* currentFrameSizeStats = + &(currentPayloadStr->frameSizeStats[k]); + currentFrameSizeStats->frameSizeSample = (int16_t)_lastFrameSizeSample; + + // increment the number of encoded samples. + currentFrameSizeStats->totalEncodedSamples += _lastFrameSizeSample; + // increment the number of recveived packets + currentFrameSizeStats->numPackets++; + // increment the total number of bytes (this is based on + // the previous payload we don't know the frame-size of + // the current payload. + currentFrameSizeStats->totalPayloadLenByte += + currentPayloadStr->lastPayloadLenByte; + // store the maximum payload-size (this is based on + // the previous payload we don't know the frame-size of + // the current payload. + if (currentFrameSizeStats->maxPayloadLen < + currentPayloadStr->lastPayloadLenByte) { + currentFrameSizeStats->maxPayloadLen = + currentPayloadStr->lastPayloadLenByte; + } + // store the current values for the next time + currentPayloadStr->lastTimestamp = rtp_header.timestamp; + currentPayloadStr->lastPayloadLenByte = payloadSize; + } else { + currentPayloadStr->newPacket = false; + currentPayloadStr->lastPayloadLenByte = payloadSize; + currentPayloadStr->lastTimestamp = rtp_header.timestamp; + currentPayloadStr->payloadType = rtp_header.payloadType; + memset(currentPayloadStr->frameSizeStats, 0, + MAX_NUM_FRAMESIZES * sizeof(ACMTestFrameSizeStats)); + } + } else { + n = 0; + while (_payloadStats[n].payloadType != -1) { + n++; + } + // first packet + _payloadStats[n].newPacket = false; + _payloadStats[n].lastPayloadLenByte = payloadSize; + _payloadStats[n].lastTimestamp = rtp_header.timestamp; + _payloadStats[n].payloadType = rtp_header.payloadType; + memset(_payloadStats[n].frameSizeStats, 0, + MAX_NUM_FRAMESIZES * sizeof(ACMTestFrameSizeStats)); + } +} + +Channel::Channel(int16_t chID) + : _receiverACM(NULL), + _seqNo(0), + _bitStreamFile(NULL), + _saveBitStream(false), + _lastPayloadType(-1), + _isStereo(false), + _leftChannel(true), + _lastInTimestamp(0), + _useLastFrameSize(false), + _lastFrameSizeSample(0), + _packetLoss(0), + _useFECTestWithPacketLoss(false), + _beginTime(rtc::TimeMillis()), + _totalBytes(0), + external_send_timestamp_(-1), + external_sequence_number_(-1), + num_packets_to_drop_(0) { + int n; + int k; + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + _payloadStats[n].payloadType = -1; + _payloadStats[n].newPacket = true; + for (k = 0; k < MAX_NUM_FRAMESIZES; k++) { + _payloadStats[n].frameSizeStats[k].frameSizeSample = 0; + _payloadStats[n].frameSizeStats[k].maxPayloadLen = 0; + _payloadStats[n].frameSizeStats[k].numPackets = 0; + _payloadStats[n].frameSizeStats[k].totalPayloadLenByte = 0; + _payloadStats[n].frameSizeStats[k].totalEncodedSamples = 0; + } + } + if (chID >= 0) { + _saveBitStream = true; + rtc::StringBuilder ss; + ss.AppendFormat("bitStream_%d.dat", chID); + _bitStreamFile = fopen(ss.str().c_str(), "wb"); + } else { + _saveBitStream = false; + } +} + +Channel::~Channel() {} + +void Channel::RegisterReceiverACM(AudioCodingModule* acm) { + _receiverACM = acm; + return; +} + +void Channel::ResetStats() { + int n; + int k; + _channelCritSect.Lock(); + _lastPayloadType = -1; + for (n = 0; n < MAX_NUM_PAYLOADS; n++) { + _payloadStats[n].payloadType = -1; + _payloadStats[n].newPacket = true; + for (k = 0; k < MAX_NUM_FRAMESIZES; k++) { + _payloadStats[n].frameSizeStats[k].frameSizeSample = 0; + _payloadStats[n].frameSizeStats[k].maxPayloadLen = 0; + _payloadStats[n].frameSizeStats[k].numPackets = 0; + _payloadStats[n].frameSizeStats[k].totalPayloadLenByte = 0; + _payloadStats[n].frameSizeStats[k].totalEncodedSamples = 0; + } + } + _beginTime = rtc::TimeMillis(); + _totalBytes = 0; + _channelCritSect.Unlock(); +} + +uint32_t Channel::LastInTimestamp() { + uint32_t timestamp; + _channelCritSect.Lock(); + timestamp = _lastInTimestamp; + _channelCritSect.Unlock(); + return timestamp; +} + +double Channel::BitRate() { + double rate; + uint64_t currTime = rtc::TimeMillis(); + _channelCritSect.Lock(); + rate = ((double)_totalBytes * 8.0) / (double)(currTime - _beginTime); + _channelCritSect.Unlock(); + return rate; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/Channel.h b/third_party/libwebrtc/modules/audio_coding/test/Channel.h new file mode 100644 index 0000000000..7a8829e1d2 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/Channel.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_CHANNEL_H_ +#define MODULES_AUDIO_CODING_TEST_CHANNEL_H_ + +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +#define MAX_NUM_PAYLOADS 50 +#define MAX_NUM_FRAMESIZES 6 + +// TODO(turajs): Write constructor for this structure. +struct ACMTestFrameSizeStats { + uint16_t frameSizeSample; + size_t maxPayloadLen; + uint32_t numPackets; + uint64_t totalPayloadLenByte; + uint64_t totalEncodedSamples; + double rateBitPerSec; + double usageLenSec; +}; + +// TODO(turajs): Write constructor for this structure. +struct ACMTestPayloadStats { + bool newPacket; + int16_t payloadType; + size_t lastPayloadLenByte; + uint32_t lastTimestamp; + ACMTestFrameSizeStats frameSizeStats[MAX_NUM_FRAMESIZES]; +}; + +class Channel : public AudioPacketizationCallback { + public: + Channel(int16_t chID = -1); + ~Channel() override; + + int32_t SendData(AudioFrameType frameType, + uint8_t payloadType, + uint32_t timeStamp, + const uint8_t* payloadData, + size_t payloadSize, + int64_t absolute_capture_timestamp_ms) override; + + void RegisterReceiverACM(AudioCodingModule* acm); + + void ResetStats(); + + void SetIsStereo(bool isStereo) { _isStereo = isStereo; } + + uint32_t LastInTimestamp(); + + void SetFECTestWithPacketLoss(bool usePacketLoss) { + _useFECTestWithPacketLoss = usePacketLoss; + } + + double BitRate(); + + void set_send_timestamp(uint32_t new_send_ts) { + external_send_timestamp_ = new_send_ts; + } + + void set_sequence_number(uint16_t new_sequence_number) { + external_sequence_number_ = new_sequence_number; + } + + void set_num_packets_to_drop(int new_num_packets_to_drop) { + num_packets_to_drop_ = new_num_packets_to_drop; + } + + private: + void CalcStatistics(const RTPHeader& rtp_header, size_t payloadSize); + + AudioCodingModule* _receiverACM; + uint16_t _seqNo; + // 60msec * 32 sample(max)/msec * 2 description (maybe) * 2 bytes/sample + uint8_t _payloadData[60 * 32 * 2 * 2]; + + Mutex _channelCritSect; + FILE* _bitStreamFile; + bool _saveBitStream; + int16_t _lastPayloadType; + ACMTestPayloadStats _payloadStats[MAX_NUM_PAYLOADS]; + bool _isStereo; + RTPHeader _rtp_header; + bool _leftChannel; + uint32_t _lastInTimestamp; + bool _useLastFrameSize; + uint32_t _lastFrameSizeSample; + // FEC Test variables + int16_t _packetLoss; + bool _useFECTestWithPacketLoss; + uint64_t _beginTime; + uint64_t _totalBytes; + + // External timing info, defaulted to -1. Only used if they are + // non-negative. + int64_t external_send_timestamp_; + int32_t external_sequence_number_; + int num_packets_to_drop_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_CHANNEL_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/EncodeDecodeTest.cc b/third_party/libwebrtc/modules/audio_coding/test/EncodeDecodeTest.cc new file mode 100644 index 0000000000..9f9c4aa74c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/EncodeDecodeTest.cc @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/EncodeDecodeTest.h" + +#include +#include + +#include + +#include "absl/strings/string_view.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +namespace { +// Buffer size for stereo 48 kHz audio. +constexpr size_t kWebRtc10MsPcmAudio = 960; + +} // namespace + +TestPacketization::TestPacketization(RTPStream* rtpStream, uint16_t frequency) + : _rtpStream(rtpStream), _frequency(frequency), _seqNo(0) {} + +TestPacketization::~TestPacketization() {} + +int32_t TestPacketization::SendData(const AudioFrameType /* frameType */, + const uint8_t payloadType, + const uint32_t timeStamp, + const uint8_t* payloadData, + const size_t payloadSize, + int64_t absolute_capture_timestamp_ms) { + _rtpStream->Write(payloadType, timeStamp, _seqNo++, payloadData, payloadSize, + _frequency); + return 1; +} + +Sender::Sender() + : _acm(NULL), _pcmFile(), _audioFrame(), _packetization(NULL) {} + +void Sender::Setup(AudioCodingModule* acm, + RTPStream* rtpStream, + absl::string_view in_file_name, + int in_sample_rate, + int payload_type, + SdpAudioFormat format) { + // Open input file + const std::string file_name = webrtc::test::ResourcePath(in_file_name, "pcm"); + _pcmFile.Open(file_name, in_sample_rate, "rb"); + if (format.num_channels == 2) { + _pcmFile.ReadStereo(true); + } + // Set test length to 500 ms (50 blocks of 10 ms each). + _pcmFile.SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the file starts with silence. + _pcmFile.FastForward(100); + + acm->SetEncoder(CreateBuiltinAudioEncoderFactory()->MakeAudioEncoder( + payload_type, format, absl::nullopt)); + _packetization = new TestPacketization(rtpStream, format.clockrate_hz); + EXPECT_EQ(0, acm->RegisterTransportCallback(_packetization)); + + _acm = acm; +} + +void Sender::Teardown() { + _pcmFile.Close(); + delete _packetization; +} + +bool Sender::Add10MsData() { + if (!_pcmFile.EndOfFile()) { + EXPECT_GT(_pcmFile.Read10MsData(_audioFrame), 0); + int32_t ok = _acm->Add10MsData(_audioFrame); + EXPECT_GE(ok, 0); + return ok >= 0 ? true : false; + } + return false; +} + +void Sender::Run() { + while (true) { + if (!Add10MsData()) { + break; + } + } +} + +Receiver::Receiver() + : _playoutLengthSmpls(kWebRtc10MsPcmAudio), + _payloadSizeBytes(MAX_INCOMING_PAYLOAD) {} + +void Receiver::Setup(AudioCodingModule* acm, + RTPStream* rtpStream, + absl::string_view out_file_name, + size_t channels, + int file_num) { + EXPECT_EQ(0, acm->InitializeReceiver()); + + if (channels == 1) { + acm->SetReceiveCodecs({{107, {"L16", 8000, 1}}, + {108, {"L16", 16000, 1}}, + {109, {"L16", 32000, 1}}, + {0, {"PCMU", 8000, 1}}, + {8, {"PCMA", 8000, 1}}, + {102, {"ILBC", 8000, 1}}, + {9, {"G722", 8000, 1}}, + {120, {"OPUS", 48000, 2}}, + {13, {"CN", 8000, 1}}, + {98, {"CN", 16000, 1}}, + {99, {"CN", 32000, 1}}}); + } else { + ASSERT_EQ(channels, 2u); + acm->SetReceiveCodecs({{111, {"L16", 8000, 2}}, + {112, {"L16", 16000, 2}}, + {113, {"L16", 32000, 2}}, + {110, {"PCMU", 8000, 2}}, + {118, {"PCMA", 8000, 2}}, + {119, {"G722", 8000, 2}}, + {120, {"OPUS", 48000, 2, {{"stereo", "1"}}}}}); + } + + int playSampFreq; + std::string file_name; + rtc::StringBuilder file_stream; + file_stream << webrtc::test::OutputPath() << out_file_name << file_num + << ".pcm"; + file_name = file_stream.str(); + _rtpStream = rtpStream; + + playSampFreq = 32000; + _pcmFile.Open(file_name, 32000, "wb+"); + + _realPayloadSizeBytes = 0; + _playoutBuffer = new int16_t[kWebRtc10MsPcmAudio]; + _frequency = playSampFreq; + _acm = acm; + _firstTime = true; +} + +void Receiver::Teardown() { + delete[] _playoutBuffer; + _pcmFile.Close(); +} + +bool Receiver::IncomingPacket() { + if (!_rtpStream->EndOfFile()) { + if (_firstTime) { + _firstTime = false; + _realPayloadSizeBytes = _rtpStream->Read(&_rtpHeader, _incomingPayload, + _payloadSizeBytes, &_nextTime); + if (_realPayloadSizeBytes == 0) { + if (_rtpStream->EndOfFile()) { + _firstTime = true; + return true; + } else { + return false; + } + } + } + + EXPECT_EQ(0, _acm->IncomingPacket(_incomingPayload, _realPayloadSizeBytes, + _rtpHeader)); + _realPayloadSizeBytes = _rtpStream->Read(&_rtpHeader, _incomingPayload, + _payloadSizeBytes, &_nextTime); + if (_realPayloadSizeBytes == 0 && _rtpStream->EndOfFile()) { + _firstTime = true; + } + } + return true; +} + +bool Receiver::PlayoutData() { + AudioFrame audioFrame; + bool muted; + int32_t ok = _acm->PlayoutData10Ms(_frequency, &audioFrame, &muted); + if (muted) { + ADD_FAILURE(); + return false; + } + EXPECT_EQ(0, ok); + if (ok < 0) { + return false; + } + if (_playoutLengthSmpls == 0) { + return false; + } + _pcmFile.Write10MsData(audioFrame.data(), audioFrame.samples_per_channel_ * + audioFrame.num_channels_); + return true; +} + +void Receiver::Run() { + uint8_t counter500Ms = 50; + uint32_t clock = 0; + + while (counter500Ms > 0) { + if (clock == 0 || clock >= _nextTime) { + EXPECT_TRUE(IncomingPacket()); + if (clock == 0) { + clock = _nextTime; + } + } + if ((clock % 10) == 0) { + if (!PlayoutData()) { + clock++; + continue; + } + } + if (_rtpStream->EndOfFile()) { + counter500Ms--; + } + clock++; + } +} + +EncodeDecodeTest::EncodeDecodeTest() = default; + +void EncodeDecodeTest::Perform() { + const std::map send_codecs = { + {107, {"L16", 8000, 1}}, {108, {"L16", 16000, 1}}, + {109, {"L16", 32000, 1}}, {0, {"PCMU", 8000, 1}}, + {8, {"PCMA", 8000, 1}}, +#ifdef WEBRTC_CODEC_ILBC + {102, {"ILBC", 8000, 1}}, +#endif + {9, {"G722", 8000, 1}}}; + int file_num = 0; + for (const auto& send_codec : send_codecs) { + RTPFile rtpFile; + std::unique_ptr acm(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))); + + std::string fileName = webrtc::test::TempFilename( + webrtc::test::OutputPath(), "encode_decode_rtp"); + rtpFile.Open(fileName.c_str(), "wb+"); + rtpFile.WriteHeader(); + Sender sender; + sender.Setup(acm.get(), &rtpFile, "audio_coding/testfile32kHz", 32000, + send_codec.first, send_codec.second); + sender.Run(); + sender.Teardown(); + rtpFile.Close(); + + rtpFile.Open(fileName.c_str(), "rb"); + rtpFile.ReadHeader(); + Receiver receiver; + receiver.Setup(acm.get(), &rtpFile, "encodeDecode_out", 1, file_num); + receiver.Run(); + receiver.Teardown(); + rtpFile.Close(); + + file_num++; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/EncodeDecodeTest.h b/third_party/libwebrtc/modules/audio_coding/test/EncodeDecodeTest.h new file mode 100644 index 0000000000..89b76440ef --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/EncodeDecodeTest.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_ENCODEDECODETEST_H_ +#define MODULES_AUDIO_CODING_TEST_ENCODEDECODETEST_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/RTPFile.h" +#include "modules/include/module_common_types.h" + +namespace webrtc { + +#define MAX_INCOMING_PAYLOAD 8096 + +// TestPacketization callback which writes the encoded payloads to file +class TestPacketization : public AudioPacketizationCallback { + public: + TestPacketization(RTPStream* rtpStream, uint16_t frequency); + ~TestPacketization(); + int32_t SendData(AudioFrameType frameType, + uint8_t payloadType, + uint32_t timeStamp, + const uint8_t* payloadData, + size_t payloadSize, + int64_t absolute_capture_timestamp_ms) override; + + private: + static void MakeRTPheader(uint8_t* rtpHeader, + uint8_t payloadType, + int16_t seqNo, + uint32_t timeStamp, + uint32_t ssrc); + RTPStream* _rtpStream; + int32_t _frequency; + int16_t _seqNo; +}; + +class Sender { + public: + Sender(); + void Setup(AudioCodingModule* acm, + RTPStream* rtpStream, + absl::string_view in_file_name, + int in_sample_rate, + int payload_type, + SdpAudioFormat format); + void Teardown(); + void Run(); + bool Add10MsData(); + + protected: + AudioCodingModule* _acm; + + private: + PCMFile _pcmFile; + AudioFrame _audioFrame; + TestPacketization* _packetization; +}; + +class Receiver { + public: + Receiver(); + virtual ~Receiver() {} + void Setup(AudioCodingModule* acm, + RTPStream* rtpStream, + absl::string_view out_file_name, + size_t channels, + int file_num); + void Teardown(); + void Run(); + virtual bool IncomingPacket(); + bool PlayoutData(); + + private: + PCMFile _pcmFile; + int16_t* _playoutBuffer; + uint16_t _playoutLengthSmpls; + int32_t _frequency; + bool _firstTime; + + protected: + AudioCodingModule* _acm; + uint8_t _incomingPayload[MAX_INCOMING_PAYLOAD]; + RTPStream* _rtpStream; + RTPHeader _rtpHeader; + size_t _realPayloadSizeBytes; + size_t _payloadSizeBytes; + uint32_t _nextTime; +}; + +class EncodeDecodeTest { + public: + EncodeDecodeTest(); + void Perform(); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_ENCODEDECODETEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/PCMFile.cc b/third_party/libwebrtc/modules/audio_coding/test/PCMFile.cc new file mode 100644 index 0000000000..e069a42de1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/PCMFile.cc @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/PCMFile.h" + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +namespace webrtc { + +#define MAX_FILE_NAME_LENGTH_BYTE 500 + +PCMFile::PCMFile() + : pcm_file_(NULL), + samples_10ms_(160), + frequency_(16000), + end_of_file_(false), + auto_rewind_(false), + rewinded_(false), + read_stereo_(false), + save_stereo_(false) { + timestamp_ = + (((uint32_t)rand() & 0x0000FFFF) << 16) | ((uint32_t)rand() & 0x0000FFFF); +} + +PCMFile::PCMFile(uint32_t timestamp) + : pcm_file_(NULL), + samples_10ms_(160), + frequency_(16000), + end_of_file_(false), + auto_rewind_(false), + rewinded_(false), + read_stereo_(false), + save_stereo_(false) { + timestamp_ = timestamp; +} + +PCMFile::~PCMFile() { + if (pcm_file_) { + fclose(pcm_file_); + } +} + +int16_t PCMFile::ChooseFile(std::string* file_name, + int16_t max_len, + uint16_t* frequency_hz) { + char tmp_name[MAX_FILE_NAME_LENGTH_BYTE]; + + EXPECT_TRUE(fgets(tmp_name, MAX_FILE_NAME_LENGTH_BYTE, stdin) != NULL); + tmp_name[MAX_FILE_NAME_LENGTH_BYTE - 1] = '\0'; + int16_t n = 0; + + // Removing trailing spaces. + while ((isspace(static_cast(tmp_name[n])) || + iscntrl(static_cast(tmp_name[n]))) && + (static_cast(tmp_name[n]) != 0) && + (n < MAX_FILE_NAME_LENGTH_BYTE)) { + n++; + } + if (n > 0) { + memmove(tmp_name, &tmp_name[n], MAX_FILE_NAME_LENGTH_BYTE - n); + } + + // Removing trailing spaces. + n = (int16_t)(strlen(tmp_name) - 1); + if (n >= 0) { + while ((isspace(static_cast(tmp_name[n])) || + iscntrl(static_cast(tmp_name[n]))) && + (n >= 0)) { + n--; + } + } + if (n >= 0) { + tmp_name[n + 1] = '\0'; + } + + int16_t len = (int16_t)strlen(tmp_name); + if (len > max_len) { + return -1; + } + if (len > 0) { + std::string tmp_string(tmp_name, len + 1); + *file_name = tmp_string; + } + printf("Enter the sampling frequency (in Hz) of the above file [%u]: ", + *frequency_hz); + EXPECT_TRUE(fgets(tmp_name, 10, stdin) != NULL); + uint16_t tmp_frequency = (uint16_t)atoi(tmp_name); + if (tmp_frequency > 0) { + *frequency_hz = tmp_frequency; + } + return 0; +} + +void PCMFile::Open(absl::string_view file_name, + uint16_t frequency, + absl::string_view mode, + bool auto_rewind) { + if ((pcm_file_ = fopen(std::string(file_name).c_str(), + std::string(mode).c_str())) == NULL) { + printf("Cannot open file %s.\n", std::string(file_name).c_str()); + ADD_FAILURE() << "Unable to read file"; + } + frequency_ = frequency; + samples_10ms_ = (uint16_t)(frequency_ / 100); + auto_rewind_ = auto_rewind; + end_of_file_ = false; + rewinded_ = false; +} + +int32_t PCMFile::SamplingFrequency() const { + return frequency_; +} + +uint16_t PCMFile::PayloadLength10Ms() const { + return samples_10ms_; +} + +int32_t PCMFile::Read10MsData(AudioFrame& audio_frame) { + uint16_t channels = 1; + if (read_stereo_) { + channels = 2; + } + + int32_t payload_size = + (int32_t)fread(audio_frame.mutable_data(), sizeof(uint16_t), + samples_10ms_ * channels, pcm_file_); + if (payload_size < samples_10ms_ * channels) { + int16_t* frame_data = audio_frame.mutable_data(); + for (int k = payload_size; k < samples_10ms_ * channels; k++) { + frame_data[k] = 0; + } + if (auto_rewind_) { + rewind(pcm_file_); + rewinded_ = true; + } else { + end_of_file_ = true; + } + } + audio_frame.samples_per_channel_ = samples_10ms_; + audio_frame.sample_rate_hz_ = frequency_; + audio_frame.num_channels_ = channels; + audio_frame.timestamp_ = timestamp_; + timestamp_ += samples_10ms_; + ++blocks_read_; + if (num_10ms_blocks_to_read_ && blocks_read_ >= *num_10ms_blocks_to_read_) + end_of_file_ = true; + return samples_10ms_; +} + +void PCMFile::Write10MsData(const AudioFrame& audio_frame) { + if (audio_frame.num_channels_ == 1) { + if (!save_stereo_) { + if (fwrite(audio_frame.data(), sizeof(uint16_t), + audio_frame.samples_per_channel_, pcm_file_) != + static_cast(audio_frame.samples_per_channel_)) { + return; + } + } else { + const int16_t* frame_data = audio_frame.data(); + int16_t* stereo_audio = new int16_t[2 * audio_frame.samples_per_channel_]; + for (size_t k = 0; k < audio_frame.samples_per_channel_; k++) { + stereo_audio[k << 1] = frame_data[k]; + stereo_audio[(k << 1) + 1] = frame_data[k]; + } + if (fwrite(stereo_audio, sizeof(int16_t), + 2 * audio_frame.samples_per_channel_, pcm_file_) != + static_cast(2 * audio_frame.samples_per_channel_)) { + return; + } + delete[] stereo_audio; + } + } else { + if (fwrite(audio_frame.data(), sizeof(int16_t), + audio_frame.num_channels_ * audio_frame.samples_per_channel_, + pcm_file_) != + static_cast(audio_frame.num_channels_ * + audio_frame.samples_per_channel_)) { + return; + } + } +} + +void PCMFile::Write10MsData(const int16_t* playout_buffer, + size_t length_smpls) { + if (fwrite(playout_buffer, sizeof(uint16_t), length_smpls, pcm_file_) != + length_smpls) { + return; + } +} + +void PCMFile::Close() { + fclose(pcm_file_); + pcm_file_ = NULL; + blocks_read_ = 0; +} + +void PCMFile::FastForward(int num_10ms_blocks) { + const int channels = read_stereo_ ? 2 : 1; + long num_bytes_to_move = + num_10ms_blocks * sizeof(int16_t) * samples_10ms_ * channels; + int error = fseek(pcm_file_, num_bytes_to_move, SEEK_CUR); + RTC_DCHECK_EQ(error, 0); +} + +void PCMFile::Rewind() { + rewind(pcm_file_); + end_of_file_ = false; + blocks_read_ = 0; +} + +bool PCMFile::Rewinded() { + return rewinded_; +} + +void PCMFile::SaveStereo(bool is_stereo) { + save_stereo_ = is_stereo; +} + +void PCMFile::ReadStereo(bool is_stereo) { + read_stereo_ = is_stereo; +} + +void PCMFile::SetNum10MsBlocksToRead(int value) { + num_10ms_blocks_to_read_ = value; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/PCMFile.h b/third_party/libwebrtc/modules/audio_coding/test/PCMFile.h new file mode 100644 index 0000000000..5320aa63d0 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/PCMFile.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_PCMFILE_H_ +#define MODULES_AUDIO_CODING_TEST_PCMFILE_H_ + +#include +#include + +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/audio/audio_frame.h" + +namespace webrtc { + +class PCMFile { + public: + PCMFile(); + PCMFile(uint32_t timestamp); + ~PCMFile(); + + void Open(absl::string_view filename, + uint16_t frequency, + absl::string_view mode, + bool auto_rewind = false); + + int32_t Read10MsData(AudioFrame& audio_frame); + + void Write10MsData(const int16_t* playout_buffer, size_t length_smpls); + void Write10MsData(const AudioFrame& audio_frame); + + uint16_t PayloadLength10Ms() const; + int32_t SamplingFrequency() const; + void Close(); + bool EndOfFile() const { return end_of_file_; } + // Moves forward the specified number of 10 ms blocks. If a limit has been set + // with SetNum10MsBlocksToRead, fast-forwarding does not count towards this + // limit. + void FastForward(int num_10ms_blocks); + void Rewind(); + static int16_t ChooseFile(std::string* file_name, + int16_t max_len, + uint16_t* frequency_hz); + bool Rewinded(); + void SaveStereo(bool is_stereo = true); + void ReadStereo(bool is_stereo = true); + // If set, the reading will stop after the specified number of blocks have + // been read. When that has happened, EndOfFile() will return true. Calling + // Rewind() will reset the counter and start over. + void SetNum10MsBlocksToRead(int value); + + private: + FILE* pcm_file_; + uint16_t samples_10ms_; + int32_t frequency_; + bool end_of_file_; + bool auto_rewind_; + bool rewinded_; + uint32_t timestamp_; + bool read_stereo_; + bool save_stereo_; + absl::optional num_10ms_blocks_to_read_; + int blocks_read_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_PCMFILE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/PacketLossTest.cc b/third_party/libwebrtc/modules/audio_coding/test/PacketLossTest.cc new file mode 100644 index 0000000000..799e9c5b1f --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/PacketLossTest.cc @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/PacketLossTest.h" + +#include + +#include "absl/strings/string_view.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +ReceiverWithPacketLoss::ReceiverWithPacketLoss() + : loss_rate_(0), + burst_length_(1), + packet_counter_(0), + lost_packet_counter_(0), + burst_lost_counter_(burst_length_) {} + +void ReceiverWithPacketLoss::Setup(AudioCodingModule* acm, + RTPStream* rtpStream, + absl::string_view out_file_name, + int channels, + int file_num, + int loss_rate, + int burst_length) { + loss_rate_ = loss_rate; + burst_length_ = burst_length; + burst_lost_counter_ = burst_length_; // To prevent first packet gets lost. + rtc::StringBuilder ss; + ss << out_file_name << "_" << loss_rate_ << "_" << burst_length_ << "_"; + Receiver::Setup(acm, rtpStream, ss.str(), channels, file_num); +} + +bool ReceiverWithPacketLoss::IncomingPacket() { + if (!_rtpStream->EndOfFile()) { + if (packet_counter_ == 0) { + _realPayloadSizeBytes = _rtpStream->Read(&_rtpHeader, _incomingPayload, + _payloadSizeBytes, &_nextTime); + if (_realPayloadSizeBytes == 0) { + if (_rtpStream->EndOfFile()) { + packet_counter_ = 0; + return true; + } else { + return false; + } + } + } + + if (!PacketLost()) { + _acm->IncomingPacket(_incomingPayload, _realPayloadSizeBytes, _rtpHeader); + } + packet_counter_++; + _realPayloadSizeBytes = _rtpStream->Read(&_rtpHeader, _incomingPayload, + _payloadSizeBytes, &_nextTime); + if (_realPayloadSizeBytes == 0 && _rtpStream->EndOfFile()) { + packet_counter_ = 0; + lost_packet_counter_ = 0; + } + } + return true; +} + +bool ReceiverWithPacketLoss::PacketLost() { + if (burst_lost_counter_ < burst_length_) { + lost_packet_counter_++; + burst_lost_counter_++; + return true; + } + + if (lost_packet_counter_ * 100 < loss_rate_ * packet_counter_) { + lost_packet_counter_++; + burst_lost_counter_ = 1; + return true; + } + return false; +} + +SenderWithFEC::SenderWithFEC() : expected_loss_rate_(0) {} + +void SenderWithFEC::Setup(AudioCodingModule* acm, + RTPStream* rtpStream, + absl::string_view in_file_name, + int payload_type, + SdpAudioFormat format, + int expected_loss_rate) { + Sender::Setup(acm, rtpStream, in_file_name, format.clockrate_hz, payload_type, + format); + EXPECT_TRUE(SetFEC(true)); + EXPECT_TRUE(SetPacketLossRate(expected_loss_rate)); +} + +bool SenderWithFEC::SetFEC(bool enable_fec) { + bool success = false; + _acm->ModifyEncoder([&](std::unique_ptr* enc) { + if (*enc && (*enc)->SetFec(enable_fec)) { + success = true; + } + }); + return success; +} + +bool SenderWithFEC::SetPacketLossRate(int expected_loss_rate) { + if (_acm->SetPacketLossRate(expected_loss_rate) == 0) { + expected_loss_rate_ = expected_loss_rate; + return true; + } + return false; +} + +PacketLossTest::PacketLossTest(int channels, + int expected_loss_rate, + int actual_loss_rate, + int burst_length) + : channels_(channels), + in_file_name_(channels_ == 1 ? "audio_coding/testfile32kHz" + : "audio_coding/teststereo32kHz"), + sample_rate_hz_(32000), + expected_loss_rate_(expected_loss_rate), + actual_loss_rate_(actual_loss_rate), + burst_length_(burst_length) {} + +void PacketLossTest::Perform() { +#ifndef WEBRTC_CODEC_OPUS + return; +#else + RTPFile rtpFile; + std::unique_ptr acm(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))); + SdpAudioFormat send_format = SdpAudioFormat("opus", 48000, 2); + if (channels_ == 2) { + send_format.parameters = {{"stereo", "1"}}; + } + + std::string fileName = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "packet_loss_test"); + rtpFile.Open(fileName.c_str(), "wb+"); + rtpFile.WriteHeader(); + SenderWithFEC sender; + sender.Setup(acm.get(), &rtpFile, in_file_name_, 120, send_format, + expected_loss_rate_); + sender.Run(); + sender.Teardown(); + rtpFile.Close(); + + rtpFile.Open(fileName.c_str(), "rb"); + rtpFile.ReadHeader(); + ReceiverWithPacketLoss receiver; + receiver.Setup(acm.get(), &rtpFile, "packetLoss_out", channels_, 15, + actual_loss_rate_, burst_length_); + receiver.Run(); + receiver.Teardown(); + rtpFile.Close(); +#endif +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/PacketLossTest.h b/third_party/libwebrtc/modules/audio_coding/test/PacketLossTest.h new file mode 100644 index 0000000000..d841d65a1b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/PacketLossTest.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_PACKETLOSSTEST_H_ +#define MODULES_AUDIO_CODING_TEST_PACKETLOSSTEST_H_ + +#include + +#include "absl/strings/string_view.h" +#include "modules/audio_coding/test/EncodeDecodeTest.h" + +namespace webrtc { + +class ReceiverWithPacketLoss : public Receiver { + public: + ReceiverWithPacketLoss(); + void Setup(AudioCodingModule* acm, + RTPStream* rtpStream, + absl::string_view out_file_name, + int channels, + int file_num, + int loss_rate, + int burst_length); + bool IncomingPacket() override; + + protected: + bool PacketLost(); + int loss_rate_; + int burst_length_; + int packet_counter_; + int lost_packet_counter_; + int burst_lost_counter_; +}; + +class SenderWithFEC : public Sender { + public: + SenderWithFEC(); + void Setup(AudioCodingModule* acm, + RTPStream* rtpStream, + absl::string_view in_file_name, + int payload_type, + SdpAudioFormat format, + int expected_loss_rate); + bool SetPacketLossRate(int expected_loss_rate); + bool SetFEC(bool enable_fec); + + protected: + int expected_loss_rate_; +}; + +class PacketLossTest { + public: + PacketLossTest(int channels, + int expected_loss_rate_, + int actual_loss_rate, + int burst_length); + void Perform(); + + protected: + int channels_; + std::string in_file_name_; + int sample_rate_hz_; + int expected_loss_rate_; + int actual_loss_rate_; + int burst_length_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_PACKETLOSSTEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/RTPFile.cc b/third_party/libwebrtc/modules/audio_coding/test/RTPFile.cc new file mode 100644 index 0000000000..0c2ab3c443 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/RTPFile.cc @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "RTPFile.h" + +#include + +#include + +#include "absl/strings/string_view.h" + +#ifdef WIN32 +#include +#else +#include +#endif + +// TODO(tlegrand): Consider removing usage of gtest. +#include "test/gtest.h" + +namespace webrtc { + +void RTPStream::ParseRTPHeader(RTPHeader* rtp_header, + const uint8_t* rtpHeader) { + rtp_header->payloadType = rtpHeader[1]; + rtp_header->sequenceNumber = + (static_cast(rtpHeader[2]) << 8) | rtpHeader[3]; + rtp_header->timestamp = (static_cast(rtpHeader[4]) << 24) | + (static_cast(rtpHeader[5]) << 16) | + (static_cast(rtpHeader[6]) << 8) | + rtpHeader[7]; + rtp_header->ssrc = (static_cast(rtpHeader[8]) << 24) | + (static_cast(rtpHeader[9]) << 16) | + (static_cast(rtpHeader[10]) << 8) | + rtpHeader[11]; +} + +void RTPStream::MakeRTPheader(uint8_t* rtpHeader, + uint8_t payloadType, + int16_t seqNo, + uint32_t timeStamp, + uint32_t ssrc) { + rtpHeader[0] = 0x80; + rtpHeader[1] = payloadType; + rtpHeader[2] = (seqNo >> 8) & 0xFF; + rtpHeader[3] = seqNo & 0xFF; + rtpHeader[4] = timeStamp >> 24; + rtpHeader[5] = (timeStamp >> 16) & 0xFF; + rtpHeader[6] = (timeStamp >> 8) & 0xFF; + rtpHeader[7] = timeStamp & 0xFF; + rtpHeader[8] = ssrc >> 24; + rtpHeader[9] = (ssrc >> 16) & 0xFF; + rtpHeader[10] = (ssrc >> 8) & 0xFF; + rtpHeader[11] = ssrc & 0xFF; +} + +RTPPacket::RTPPacket(uint8_t payloadType, + uint32_t timeStamp, + int16_t seqNo, + const uint8_t* payloadData, + size_t payloadSize, + uint32_t frequency) + : payloadType(payloadType), + timeStamp(timeStamp), + seqNo(seqNo), + payloadSize(payloadSize), + frequency(frequency) { + if (payloadSize > 0) { + this->payloadData = new uint8_t[payloadSize]; + memcpy(this->payloadData, payloadData, payloadSize); + } +} + +RTPPacket::~RTPPacket() { + delete[] payloadData; +} + +void RTPBuffer::Write(const uint8_t payloadType, + const uint32_t timeStamp, + const int16_t seqNo, + const uint8_t* payloadData, + const size_t payloadSize, + uint32_t frequency) { + RTPPacket* packet = new RTPPacket(payloadType, timeStamp, seqNo, payloadData, + payloadSize, frequency); + MutexLock lock(&mutex_); + _rtpQueue.push(packet); +} + +size_t RTPBuffer::Read(RTPHeader* rtp_header, + uint8_t* payloadData, + size_t payloadSize, + uint32_t* offset) { + RTPPacket* packet; + { + MutexLock lock(&mutex_); + packet = _rtpQueue.front(); + _rtpQueue.pop(); + } + rtp_header->markerBit = 1; + rtp_header->payloadType = packet->payloadType; + rtp_header->sequenceNumber = packet->seqNo; + rtp_header->ssrc = 0; + rtp_header->timestamp = packet->timeStamp; + if (packet->payloadSize > 0 && payloadSize >= packet->payloadSize) { + memcpy(payloadData, packet->payloadData, packet->payloadSize); + } else { + return 0; + } + *offset = (packet->timeStamp / (packet->frequency / 1000)); + + return packet->payloadSize; +} + +bool RTPBuffer::EndOfFile() const { + MutexLock lock(&mutex_); + return _rtpQueue.empty(); +} + +void RTPFile::Open(absl::string_view filename, absl::string_view mode) { + std::string filename_str = std::string(filename); + if ((_rtpFile = fopen(filename_str.c_str(), std::string(mode).c_str())) == + NULL) { + printf("Cannot write file %s.\n", filename_str.c_str()); + ADD_FAILURE() << "Unable to write file"; + exit(1); + } +} + +void RTPFile::Close() { + if (_rtpFile != NULL) { + fclose(_rtpFile); + _rtpFile = NULL; + } +} + +void RTPFile::WriteHeader() { + // Write data in a format that NetEQ and RTP Play can parse + fprintf(_rtpFile, "#!RTPencode%s\n", "1.0"); + uint32_t dummy_variable = 0; + // should be converted to network endian format, but does not matter when 0 + EXPECT_EQ(1u, fwrite(&dummy_variable, 4, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&dummy_variable, 4, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&dummy_variable, 4, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&dummy_variable, 2, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&dummy_variable, 2, 1, _rtpFile)); + fflush(_rtpFile); +} + +void RTPFile::ReadHeader() { + uint32_t start_sec, start_usec, source; + uint16_t port, padding; + char fileHeader[40]; + EXPECT_TRUE(fgets(fileHeader, 40, _rtpFile) != 0); + EXPECT_EQ(1u, fread(&start_sec, 4, 1, _rtpFile)); + start_sec = ntohl(start_sec); + EXPECT_EQ(1u, fread(&start_usec, 4, 1, _rtpFile)); + start_usec = ntohl(start_usec); + EXPECT_EQ(1u, fread(&source, 4, 1, _rtpFile)); + source = ntohl(source); + EXPECT_EQ(1u, fread(&port, 2, 1, _rtpFile)); + port = ntohs(port); + EXPECT_EQ(1u, fread(&padding, 2, 1, _rtpFile)); + padding = ntohs(padding); +} + +void RTPFile::Write(const uint8_t payloadType, + const uint32_t timeStamp, + const int16_t seqNo, + const uint8_t* payloadData, + const size_t payloadSize, + uint32_t frequency) { + /* write RTP packet to file */ + uint8_t rtpHeader[12]; + MakeRTPheader(rtpHeader, payloadType, seqNo, timeStamp, 0); + ASSERT_LE(12 + payloadSize + 8, std::numeric_limits::max()); + uint16_t lengthBytes = htons(static_cast(12 + payloadSize + 8)); + uint16_t plen = htons(static_cast(12 + payloadSize)); + uint32_t offsetMs; + + offsetMs = (timeStamp / (frequency / 1000)); + offsetMs = htonl(offsetMs); + EXPECT_EQ(1u, fwrite(&lengthBytes, 2, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&plen, 2, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&offsetMs, 4, 1, _rtpFile)); + EXPECT_EQ(1u, fwrite(&rtpHeader, 12, 1, _rtpFile)); + EXPECT_EQ(payloadSize, fwrite(payloadData, 1, payloadSize, _rtpFile)); +} + +size_t RTPFile::Read(RTPHeader* rtp_header, + uint8_t* payloadData, + size_t payloadSize, + uint32_t* offset) { + uint16_t lengthBytes; + uint16_t plen; + uint8_t rtpHeader[12]; + size_t read_len = fread(&lengthBytes, 2, 1, _rtpFile); + /* Check if we have reached end of file. */ + if ((read_len == 0) && feof(_rtpFile)) { + _rtpEOF = true; + return 0; + } + EXPECT_EQ(1u, fread(&plen, 2, 1, _rtpFile)); + EXPECT_EQ(1u, fread(offset, 4, 1, _rtpFile)); + lengthBytes = ntohs(lengthBytes); + plen = ntohs(plen); + *offset = ntohl(*offset); + EXPECT_GT(plen, 11); + + EXPECT_EQ(1u, fread(rtpHeader, 12, 1, _rtpFile)); + ParseRTPHeader(rtp_header, rtpHeader); + EXPECT_EQ(lengthBytes, plen + 8); + + if (plen == 0) { + return 0; + } + if (lengthBytes < 20) { + return 0; + } + if (payloadSize < static_cast((lengthBytes - 20))) { + return 0; + } + lengthBytes -= 20; + EXPECT_EQ(lengthBytes, fread(payloadData, 1, lengthBytes, _rtpFile)); + return lengthBytes; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/RTPFile.h b/third_party/libwebrtc/modules/audio_coding/test/RTPFile.h new file mode 100644 index 0000000000..b796491da9 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/RTPFile.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_RTPFILE_H_ +#define MODULES_AUDIO_CODING_TEST_RTPFILE_H_ + +#include + +#include + +#include "absl/strings/string_view.h" +#include "api/rtp_headers.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +class RTPStream { + public: + virtual ~RTPStream() {} + + virtual void Write(uint8_t payloadType, + uint32_t timeStamp, + int16_t seqNo, + const uint8_t* payloadData, + size_t payloadSize, + uint32_t frequency) = 0; + + // Returns the packet's payload size. Zero should be treated as an + // end-of-stream (in the case that EndOfFile() is true) or an error. + virtual size_t Read(RTPHeader* rtp_Header, + uint8_t* payloadData, + size_t payloadSize, + uint32_t* offset) = 0; + virtual bool EndOfFile() const = 0; + + protected: + void MakeRTPheader(uint8_t* rtpHeader, + uint8_t payloadType, + int16_t seqNo, + uint32_t timeStamp, + uint32_t ssrc); + + void ParseRTPHeader(RTPHeader* rtp_header, const uint8_t* rtpHeader); +}; + +class RTPPacket { + public: + RTPPacket(uint8_t payloadType, + uint32_t timeStamp, + int16_t seqNo, + const uint8_t* payloadData, + size_t payloadSize, + uint32_t frequency); + + ~RTPPacket(); + + uint8_t payloadType; + uint32_t timeStamp; + int16_t seqNo; + uint8_t* payloadData; + size_t payloadSize; + uint32_t frequency; +}; + +class RTPBuffer : public RTPStream { + public: + RTPBuffer() = default; + + ~RTPBuffer() = default; + + void Write(uint8_t payloadType, + uint32_t timeStamp, + int16_t seqNo, + const uint8_t* payloadData, + size_t payloadSize, + uint32_t frequency) override; + + size_t Read(RTPHeader* rtp_header, + uint8_t* payloadData, + size_t payloadSize, + uint32_t* offset) override; + + bool EndOfFile() const override; + + private: + mutable Mutex mutex_; + std::queue _rtpQueue RTC_GUARDED_BY(&mutex_); +}; + +class RTPFile : public RTPStream { + public: + ~RTPFile() {} + + RTPFile() : _rtpFile(NULL), _rtpEOF(false) {} + + void Open(absl::string_view outFilename, absl::string_view mode); + + void Close(); + + void WriteHeader(); + + void ReadHeader(); + + void Write(uint8_t payloadType, + uint32_t timeStamp, + int16_t seqNo, + const uint8_t* payloadData, + size_t payloadSize, + uint32_t frequency) override; + + size_t Read(RTPHeader* rtp_header, + uint8_t* payloadData, + size_t payloadSize, + uint32_t* offset) override; + + bool EndOfFile() const override { return _rtpEOF; } + + private: + FILE* _rtpFile; + bool _rtpEOF; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_RTPFILE_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestAllCodecs.cc b/third_party/libwebrtc/modules/audio_coding/test/TestAllCodecs.cc new file mode 100644 index 0000000000..b44037d732 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TestAllCodecs.cc @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/TestAllCodecs.h" + +#include +#include +#include + +#include "absl/strings/match.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/logging.h" +#include "rtc_base/string_encode.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +// Description of the test: +// In this test we set up a one-way communication channel from a participant +// called "a" to a participant called "b". +// a -> channel_a_to_b -> b +// +// The test loops through all available mono codecs, encode at "a" sends over +// the channel, and decodes at "b". + +#define CHECK_ERROR(f) \ + do { \ + EXPECT_GE(f, 0) << "Error Calling API"; \ + } while (0) + +namespace { +const size_t kVariableSize = std::numeric_limits::max(); +} + +namespace webrtc { + +// Class for simulating packet handling. +TestPack::TestPack() + : receiver_acm_(NULL), + sequence_number_(0), + timestamp_diff_(0), + last_in_timestamp_(0), + total_bytes_(0), + payload_size_(0) {} + +TestPack::~TestPack() {} + +void TestPack::RegisterReceiverACM(AudioCodingModule* acm) { + receiver_acm_ = acm; + return; +} + +int32_t TestPack::SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms) { + RTPHeader rtp_header; + int32_t status; + + rtp_header.markerBit = false; + rtp_header.ssrc = 0; + rtp_header.sequenceNumber = sequence_number_++; + rtp_header.payloadType = payload_type; + rtp_header.timestamp = timestamp; + + if (frame_type == AudioFrameType::kEmptyFrame) { + // Skip this frame. + return 0; + } + + // Only run mono for all test cases. + memcpy(payload_data_, payload_data, payload_size); + + status = + receiver_acm_->IncomingPacket(payload_data_, payload_size, rtp_header); + + payload_size_ = payload_size; + timestamp_diff_ = timestamp - last_in_timestamp_; + last_in_timestamp_ = timestamp; + total_bytes_ += payload_size; + return status; +} + +size_t TestPack::payload_size() { + return payload_size_; +} + +uint32_t TestPack::timestamp_diff() { + return timestamp_diff_; +} + +void TestPack::reset_payload_size() { + payload_size_ = 0; +} + +TestAllCodecs::TestAllCodecs() + : acm_a_(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))), + acm_b_(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))), + channel_a_to_b_(NULL), + test_count_(0), + packet_size_samples_(0), + packet_size_bytes_(0) {} + +TestAllCodecs::~TestAllCodecs() { + if (channel_a_to_b_ != NULL) { + delete channel_a_to_b_; + channel_a_to_b_ = NULL; + } +} + +void TestAllCodecs::Perform() { + const std::string file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + infile_a_.Open(file_name, 32000, "rb"); + + acm_a_->InitializeReceiver(); + acm_b_->InitializeReceiver(); + + acm_b_->SetReceiveCodecs({{107, {"L16", 8000, 1}}, + {108, {"L16", 16000, 1}}, + {109, {"L16", 32000, 1}}, + {111, {"L16", 8000, 2}}, + {112, {"L16", 16000, 2}}, + {113, {"L16", 32000, 2}}, + {0, {"PCMU", 8000, 1}}, + {110, {"PCMU", 8000, 2}}, + {8, {"PCMA", 8000, 1}}, + {118, {"PCMA", 8000, 2}}, + {102, {"ILBC", 8000, 1}}, + {9, {"G722", 8000, 1}}, + {119, {"G722", 8000, 2}}, + {120, {"OPUS", 48000, 2, {{"stereo", "1"}}}}, + {13, {"CN", 8000, 1}}, + {98, {"CN", 16000, 1}}, + {99, {"CN", 32000, 1}}}); + + // Create and connect the channel + channel_a_to_b_ = new TestPack; + acm_a_->RegisterTransportCallback(channel_a_to_b_); + channel_a_to_b_->RegisterReceiverACM(acm_b_.get()); + + // All codecs are tested for all allowed sampling frequencies, rates and + // packet sizes. + test_count_++; + OpenOutFile(test_count_); + char codec_g722[] = "G722"; + RegisterSendCodec('A', codec_g722, 16000, 64000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 480, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 640, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 800, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 960, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); +#ifdef WEBRTC_CODEC_ILBC + test_count_++; + OpenOutFile(test_count_); + char codec_ilbc[] = "ILBC"; + RegisterSendCodec('A', codec_ilbc, 8000, 13300, 240, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_ilbc, 8000, 13300, 480, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_ilbc, 8000, 15200, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_ilbc, 8000, 15200, 320, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); +#endif + test_count_++; + OpenOutFile(test_count_); + char codec_l16[] = "L16"; + RegisterSendCodec('A', codec_l16, 8000, 128000, 80, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 240, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 320, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); + + test_count_++; + OpenOutFile(test_count_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 480, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 640, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); + + test_count_++; + OpenOutFile(test_count_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 640, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); + + test_count_++; + OpenOutFile(test_count_); + char codec_pcma[] = "PCMA"; + RegisterSendCodec('A', codec_pcma, 8000, 64000, 80, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 240, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 400, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 480, 0); + Run(channel_a_to_b_); + + char codec_pcmu[] = "PCMU"; + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 80, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 160, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 240, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 320, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 400, 0); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 480, 0); + Run(channel_a_to_b_); + outfile_b_.Close(); +#ifdef WEBRTC_CODEC_OPUS + test_count_++; + OpenOutFile(test_count_); + char codec_opus[] = "OPUS"; + RegisterSendCodec('A', codec_opus, 48000, 6000, 480, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 20000, 480 * 2, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 32000, 480 * 4, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 48000, 480, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 64000, 480 * 4, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 96000, 480 * 6, kVariableSize); + Run(channel_a_to_b_); + RegisterSendCodec('A', codec_opus, 48000, 500000, 480 * 2, kVariableSize); + Run(channel_a_to_b_); + outfile_b_.Close(); +#endif +} + +// Register Codec to use in the test +// +// Input: side - which ACM to use, 'A' or 'B' +// codec_name - name to use when register the codec +// sampling_freq_hz - sampling frequency in Herz +// rate - bitrate in bytes +// packet_size - packet size in samples +// extra_byte - if extra bytes needed compared to the bitrate +// used when registering, can be an internal header +// set to kVariableSize if the codec is a variable +// rate codec +void TestAllCodecs::RegisterSendCodec(char side, + char* codec_name, + int32_t sampling_freq_hz, + int rate, + int packet_size, + size_t extra_byte) { + // Store packet-size in samples, used to validate the received packet. + // If G.722, store half the size to compensate for the timestamp bug in the + // RFC for G.722. + int clockrate_hz = sampling_freq_hz; + size_t num_channels = 1; + if (absl::EqualsIgnoreCase(codec_name, "G722")) { + packet_size_samples_ = packet_size / 2; + clockrate_hz = sampling_freq_hz / 2; + } else if (absl::EqualsIgnoreCase(codec_name, "OPUS")) { + packet_size_samples_ = packet_size; + num_channels = 2; + } else { + packet_size_samples_ = packet_size; + } + + // Store the expected packet size in bytes, used to validate the received + // packet. If variable rate codec (extra_byte == -1), set to -1. + if (extra_byte != kVariableSize) { + // Add 0.875 to always round up to a whole byte + packet_size_bytes_ = + static_cast(static_cast(packet_size * rate) / + static_cast(sampling_freq_hz * 8) + + 0.875) + + extra_byte; + } else { + // Packets will have a variable size. + packet_size_bytes_ = kVariableSize; + } + + // Set pointer to the ACM where to register the codec. + AudioCodingModule* my_acm = NULL; + switch (side) { + case 'A': { + my_acm = acm_a_.get(); + break; + } + case 'B': { + my_acm = acm_b_.get(); + break; + } + default: { + break; + } + } + ASSERT_TRUE(my_acm != NULL); + + auto factory = CreateBuiltinAudioEncoderFactory(); + constexpr int payload_type = 17; + SdpAudioFormat format = {codec_name, clockrate_hz, num_channels}; + format.parameters["ptime"] = rtc::ToString(rtc::CheckedDivExact( + packet_size, rtc::CheckedDivExact(sampling_freq_hz, 1000))); + my_acm->SetEncoder( + factory->MakeAudioEncoder(payload_type, format, absl::nullopt)); +} + +void TestAllCodecs::Run(TestPack* channel) { + AudioFrame audio_frame; + + int32_t out_freq_hz = outfile_b_.SamplingFrequency(); + size_t receive_size; + uint32_t timestamp_diff; + channel->reset_payload_size(); + int error_count = 0; + int counter = 0; + // Set test length to 500 ms (50 blocks of 10 ms each). + infile_a_.SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the file starts with silence. + infile_a_.FastForward(100); + + while (!infile_a_.EndOfFile()) { + // Add 10 msec to ACM. + infile_a_.Read10MsData(audio_frame); + CHECK_ERROR(acm_a_->Add10MsData(audio_frame)); + + // Verify that the received packet size matches the settings. + receive_size = channel->payload_size(); + if (receive_size) { + if ((receive_size != packet_size_bytes_) && + (packet_size_bytes_ != kVariableSize)) { + error_count++; + } + + // Verify that the timestamp is updated with expected length. The counter + // is used to avoid problems when switching codec or frame size in the + // test. + timestamp_diff = channel->timestamp_diff(); + if ((counter > 10) && + (static_cast(timestamp_diff) != packet_size_samples_) && + (packet_size_samples_ > -1)) + error_count++; + } + + // Run received side of ACM. + bool muted; + CHECK_ERROR(acm_b_->PlayoutData10Ms(out_freq_hz, &audio_frame, &muted)); + ASSERT_FALSE(muted); + + // Write output speech to file. + outfile_b_.Write10MsData(audio_frame.data(), + audio_frame.samples_per_channel_); + + // Update loop counter + counter++; + } + + EXPECT_EQ(0, error_count); + + if (infile_a_.EndOfFile()) { + infile_a_.Rewind(); + } +} + +void TestAllCodecs::OpenOutFile(int test_number) { + std::string filename = webrtc::test::OutputPath(); + rtc::StringBuilder test_number_str; + test_number_str << test_number; + filename += "testallcodecs_out_"; + filename += test_number_str.str(); + filename += ".pcm"; + outfile_b_.Open(filename, 32000, "wb"); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestAllCodecs.h b/third_party/libwebrtc/modules/audio_coding/test/TestAllCodecs.h new file mode 100644 index 0000000000..0c276414e4 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TestAllCodecs.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TESTALLCODECS_H_ +#define MODULES_AUDIO_CODING_TEST_TESTALLCODECS_H_ + +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/PCMFile.h" + +namespace webrtc { + +class TestPack : public AudioPacketizationCallback { + public: + TestPack(); + ~TestPack(); + + void RegisterReceiverACM(AudioCodingModule* acm); + + int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms) override; + + size_t payload_size(); + uint32_t timestamp_diff(); + void reset_payload_size(); + + private: + AudioCodingModule* receiver_acm_; + uint16_t sequence_number_; + uint8_t payload_data_[60 * 32 * 2 * 2]; + uint32_t timestamp_diff_; + uint32_t last_in_timestamp_; + uint64_t total_bytes_; + size_t payload_size_; +}; + +class TestAllCodecs { + public: + TestAllCodecs(); + ~TestAllCodecs(); + + void Perform(); + + private: + // The default value of '-1' indicates that the registration is based only on + // codec name, and a sampling frequency matching is not required. + // This is useful for codecs which support several sampling frequency. + // Note! Only mono mode is tested in this test. + void RegisterSendCodec(char side, + char* codec_name, + int32_t sampling_freq_hz, + int rate, + int packet_size, + size_t extra_byte); + + void Run(TestPack* channel); + void OpenOutFile(int test_number); + + std::unique_ptr acm_a_; + std::unique_ptr acm_b_; + TestPack* channel_a_to_b_; + PCMFile infile_a_; + PCMFile outfile_b_; + int test_count_; + int packet_size_samples_; + size_t packet_size_bytes_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TESTALLCODECS_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestRedFec.cc b/third_party/libwebrtc/modules/audio_coding/test/TestRedFec.cc new file mode 100644 index 0000000000..fff48b27bc --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TestRedFec.cc @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/TestRedFec.h" + +#include +#include + +#include "absl/strings/match.h" +#include "api/audio_codecs/L16/audio_decoder_L16.h" +#include "api/audio_codecs/L16/audio_encoder_L16.h" +#include "api/audio_codecs/audio_decoder_factory_template.h" +#include "api/audio_codecs/audio_encoder_factory_template.h" +#include "api/audio_codecs/g711/audio_decoder_g711.h" +#include "api/audio_codecs/g711/audio_encoder_g711.h" +#include "api/audio_codecs/g722/audio_decoder_g722.h" +#include "api/audio_codecs/g722/audio_encoder_g722.h" +#include "api/audio_codecs/opus/audio_decoder_opus.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" +#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +TestRedFec::TestRedFec() + : encoder_factory_(CreateAudioEncoderFactory()), + decoder_factory_(CreateAudioDecoderFactory()), + _acmA(AudioCodingModule::Create( + AudioCodingModule::Config(decoder_factory_))), + _acmB(AudioCodingModule::Create( + AudioCodingModule::Config(decoder_factory_))), + _channelA2B(NULL), + _testCntr(0) {} + +TestRedFec::~TestRedFec() { + if (_channelA2B != NULL) { + delete _channelA2B; + _channelA2B = NULL; + } +} + +void TestRedFec::Perform() { + const std::string file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + _inFileA.Open(file_name, 32000, "rb"); + + ASSERT_EQ(0, _acmA->InitializeReceiver()); + ASSERT_EQ(0, _acmB->InitializeReceiver()); + + // Create and connect the channel + _channelA2B = new Channel; + _acmA->RegisterTransportCallback(_channelA2B); + _channelA2B->RegisterReceiverACM(_acmB.get()); + + RegisterSendCodec(_acmA, {"L16", 8000, 1}, Vad::kVadAggressive, true); + + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + // Switch to another 8 kHz codec; RED should remain switched on. + RegisterSendCodec(_acmA, {"PCMU", 8000, 1}, Vad::kVadAggressive, true); + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + // Switch to a 16 kHz codec; RED should be switched off. + RegisterSendCodec(_acmA, {"G722", 8000, 1}, Vad::kVadAggressive, false); + + OpenOutFile(_testCntr); + RegisterSendCodec(_acmA, {"G722", 8000, 1}, Vad::kVadAggressive, false); + Run(); + RegisterSendCodec(_acmA, {"G722", 8000, 1}, Vad::kVadAggressive, false); + Run(); + _outFileB.Close(); + + _channelA2B->SetFECTestWithPacketLoss(true); + // Following tests are under packet losses. + + // Switch to a 16 kHz codec; RED should be switched off. + RegisterSendCodec(_acmA, {"G722", 8000, 1}, Vad::kVadAggressive, false); + + OpenOutFile(_testCntr); + Run(); + _outFileB.Close(); + + RegisterSendCodec(_acmA, {"opus", 48000, 2}, absl::nullopt, false); + + // _channelA2B imposes 25% packet loss rate. + EXPECT_EQ(0, _acmA->SetPacketLossRate(25)); + + _acmA->ModifyEncoder([&](std::unique_ptr* enc) { + EXPECT_EQ(true, (*enc)->SetFec(true)); + }); + + OpenOutFile(_testCntr); + Run(); + + // Switch to L16 with RED. + RegisterSendCodec(_acmA, {"L16", 8000, 1}, absl::nullopt, true); + Run(); + + // Switch to Opus again. + RegisterSendCodec(_acmA, {"opus", 48000, 2}, absl::nullopt, false); + _acmA->ModifyEncoder([&](std::unique_ptr* enc) { + EXPECT_EQ(true, (*enc)->SetFec(false)); + }); + Run(); + + _acmA->ModifyEncoder([&](std::unique_ptr* enc) { + EXPECT_EQ(true, (*enc)->SetFec(true)); + }); + _outFileB.Close(); +} + +void TestRedFec::RegisterSendCodec( + const std::unique_ptr& acm, + const SdpAudioFormat& codec_format, + absl::optional vad_mode, + bool use_red) { + constexpr int payload_type = 17, cn_payload_type = 27, red_payload_type = 37; + const auto& other_acm = &acm == &_acmA ? _acmB : _acmA; + + auto encoder = encoder_factory_->MakeAudioEncoder(payload_type, codec_format, + absl::nullopt); + EXPECT_NE(encoder, nullptr); + std::map receive_codecs = {{payload_type, codec_format}}; + if (!absl::EqualsIgnoreCase(codec_format.name, "opus")) { + if (vad_mode.has_value()) { + AudioEncoderCngConfig config; + config.speech_encoder = std::move(encoder); + config.num_channels = 1; + config.payload_type = cn_payload_type; + config.vad_mode = vad_mode.value(); + encoder = CreateComfortNoiseEncoder(std::move(config)); + receive_codecs.emplace(std::make_pair( + cn_payload_type, SdpAudioFormat("CN", codec_format.clockrate_hz, 1))); + } + if (use_red) { + AudioEncoderCopyRed::Config config; + config.payload_type = red_payload_type; + config.speech_encoder = std::move(encoder); + encoder = std::make_unique(std::move(config), + field_trials_); + receive_codecs.emplace( + std::make_pair(red_payload_type, + SdpAudioFormat("red", codec_format.clockrate_hz, 1))); + } + } + acm->SetEncoder(std::move(encoder)); + other_acm->SetReceiveCodecs(receive_codecs); +} + +void TestRedFec::Run() { + AudioFrame audioFrame; + int32_t outFreqHzB = _outFileB.SamplingFrequency(); + // Set test length to 500 ms (50 blocks of 10 ms each). + _inFileA.SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the file starts with silence. + _inFileA.FastForward(100); + + while (!_inFileA.EndOfFile()) { + EXPECT_GT(_inFileA.Read10MsData(audioFrame), 0); + EXPECT_GE(_acmA->Add10MsData(audioFrame), 0); + bool muted; + EXPECT_EQ(0, _acmB->PlayoutData10Ms(outFreqHzB, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileB.Write10MsData(audioFrame.data(), audioFrame.samples_per_channel_); + } + _inFileA.Rewind(); +} + +void TestRedFec::OpenOutFile(int16_t test_number) { + std::string file_name; + rtc::StringBuilder file_stream; + file_stream << webrtc::test::OutputPath(); + file_stream << "TestRedFec_outFile_"; + file_stream << test_number << ".pcm"; + file_name = file_stream.str(); + _outFileB.Open(file_name, 16000, "wb"); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestRedFec.h b/third_party/libwebrtc/modules/audio_coding/test/TestRedFec.h new file mode 100644 index 0000000000..dbadd88487 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TestRedFec.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TESTREDFEC_H_ +#define MODULES_AUDIO_CODING_TEST_TESTREDFEC_H_ + +#include +#include + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "common_audio/vad/include/vad.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "test/scoped_key_value_config.h" + +namespace webrtc { + +class TestRedFec { + public: + explicit TestRedFec(); + ~TestRedFec(); + + void Perform(); + + private: + void RegisterSendCodec(const std::unique_ptr& acm, + const SdpAudioFormat& codec_format, + absl::optional vad_mode, + bool use_red); + void Run(); + void OpenOutFile(int16_t testNumber); + + test::ScopedKeyValueConfig field_trials_; + const rtc::scoped_refptr encoder_factory_; + const rtc::scoped_refptr decoder_factory_; + std::unique_ptr _acmA; + std::unique_ptr _acmB; + + Channel* _channelA2B; + + PCMFile _inFileA; + PCMFile _outFileB; + int16_t _testCntr; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TESTREDFEC_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestStereo.cc b/third_party/libwebrtc/modules/audio_coding/test/TestStereo.cc new file mode 100644 index 0000000000..599fafb602 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TestStereo.cc @@ -0,0 +1,599 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/TestStereo.h" + +#include + +#include "absl/strings/match.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +// Class for simulating packet handling +TestPackStereo::TestPackStereo() + : receiver_acm_(NULL), + seq_no_(0), + timestamp_diff_(0), + last_in_timestamp_(0), + total_bytes_(0), + payload_size_(0), + lost_packet_(false) {} + +TestPackStereo::~TestPackStereo() {} + +void TestPackStereo::RegisterReceiverACM(AudioCodingModule* acm) { + receiver_acm_ = acm; + return; +} + +int32_t TestPackStereo::SendData(const AudioFrameType frame_type, + const uint8_t payload_type, + const uint32_t timestamp, + const uint8_t* payload_data, + const size_t payload_size, + int64_t absolute_capture_timestamp_ms) { + RTPHeader rtp_header; + int32_t status = 0; + + rtp_header.markerBit = false; + rtp_header.ssrc = 0; + rtp_header.sequenceNumber = seq_no_++; + rtp_header.payloadType = payload_type; + rtp_header.timestamp = timestamp; + if (frame_type == AudioFrameType::kEmptyFrame) { + // Skip this frame + return 0; + } + + if (lost_packet_ == false) { + status = + receiver_acm_->IncomingPacket(payload_data, payload_size, rtp_header); + + if (frame_type != AudioFrameType::kAudioFrameCN) { + payload_size_ = static_cast(payload_size); + } else { + payload_size_ = -1; + } + + timestamp_diff_ = timestamp - last_in_timestamp_; + last_in_timestamp_ = timestamp; + total_bytes_ += payload_size; + } + return status; +} + +uint16_t TestPackStereo::payload_size() { + return static_cast(payload_size_); +} + +uint32_t TestPackStereo::timestamp_diff() { + return timestamp_diff_; +} + +void TestPackStereo::reset_payload_size() { + payload_size_ = 0; +} + +void TestPackStereo::set_codec_mode(enum StereoMonoMode mode) { + codec_mode_ = mode; +} + +void TestPackStereo::set_lost_packet(bool lost) { + lost_packet_ = lost; +} + +TestStereo::TestStereo() + : acm_a_(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))), + acm_b_(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))), + channel_a2b_(NULL), + test_cntr_(0), + pack_size_samp_(0), + pack_size_bytes_(0), + counter_(0) {} + +TestStereo::~TestStereo() { + if (channel_a2b_ != NULL) { + delete channel_a2b_; + channel_a2b_ = NULL; + } +} + +void TestStereo::Perform() { + uint16_t frequency_hz; + int audio_channels; + int codec_channels; + + // Open both mono and stereo test files in 32 kHz. + const std::string file_name_stereo = + webrtc::test::ResourcePath("audio_coding/teststereo32kHz", "pcm"); + const std::string file_name_mono = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + frequency_hz = 32000; + in_file_stereo_ = new PCMFile(); + in_file_mono_ = new PCMFile(); + in_file_stereo_->Open(file_name_stereo, frequency_hz, "rb"); + in_file_stereo_->ReadStereo(true); + in_file_mono_->Open(file_name_mono, frequency_hz, "rb"); + in_file_mono_->ReadStereo(false); + + // Create and initialize two ACMs, one for each side of a one-to-one call. + ASSERT_TRUE((acm_a_.get() != NULL) && (acm_b_.get() != NULL)); + EXPECT_EQ(0, acm_a_->InitializeReceiver()); + EXPECT_EQ(0, acm_b_->InitializeReceiver()); + + acm_b_->SetReceiveCodecs({{103, {"ISAC", 16000, 1}}, + {104, {"ISAC", 32000, 1}}, + {107, {"L16", 8000, 1}}, + {108, {"L16", 16000, 1}}, + {109, {"L16", 32000, 1}}, + {111, {"L16", 8000, 2}}, + {112, {"L16", 16000, 2}}, + {113, {"L16", 32000, 2}}, + {0, {"PCMU", 8000, 1}}, + {110, {"PCMU", 8000, 2}}, + {8, {"PCMA", 8000, 1}}, + {118, {"PCMA", 8000, 2}}, + {102, {"ILBC", 8000, 1}}, + {9, {"G722", 8000, 1}}, + {119, {"G722", 8000, 2}}, + {120, {"OPUS", 48000, 2, {{"stereo", "1"}}}}, + {13, {"CN", 8000, 1}}, + {98, {"CN", 16000, 1}}, + {99, {"CN", 32000, 1}}}); + + // Create and connect the channel. + channel_a2b_ = new TestPackStereo; + EXPECT_EQ(0, acm_a_->RegisterTransportCallback(channel_a2b_)); + channel_a2b_->RegisterReceiverACM(acm_b_.get()); + + char codec_pcma_temp[] = "PCMA"; + RegisterSendCodec('A', codec_pcma_temp, 8000, 64000, 80, 2); + + // + // Test Stereo-To-Stereo for all codecs. + // + audio_channels = 2; + codec_channels = 2; + + // All codecs are tested for all allowed sampling frequencies, rates and + // packet sizes. + channel_a2b_->set_codec_mode(kStereo); + test_cntr_++; + OpenOutFile(test_cntr_); + char codec_g722[] = "G722"; + RegisterSendCodec('A', codec_g722, 16000, 64000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 320, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 480, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 640, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 800, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_g722, 16000, 64000, 960, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + channel_a2b_->set_codec_mode(kStereo); + test_cntr_++; + OpenOutFile(test_cntr_); + char codec_l16[] = "L16"; + RegisterSendCodec('A', codec_l16, 8000, 128000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 8000, 128000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 8000, 128000, 240, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 8000, 128000, 320, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 16000, 256000, 320, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 16000, 256000, 480, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 16000, 256000, 640, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 320, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_l16, 32000, 512000, 640, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#ifdef PCMA_AND_PCMU + channel_a2b_->set_codec_mode(kStereo); + audio_channels = 2; + codec_channels = 2; + test_cntr_++; + OpenOutFile(test_cntr_); + char codec_pcma[] = "PCMA"; + RegisterSendCodec('A', codec_pcma, 8000, 64000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 240, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 320, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 400, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 480, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + OpenOutFile(test_cntr_); + char codec_pcmu[] = "PCMU"; + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 240, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 320, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 400, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 480, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif +#ifdef WEBRTC_CODEC_OPUS + channel_a2b_->set_codec_mode(kStereo); + audio_channels = 2; + codec_channels = 2; + test_cntr_++; + OpenOutFile(test_cntr_); + + char codec_opus[] = "opus"; + // Run Opus with 10 ms frame size. + RegisterSendCodec('A', codec_opus, 48000, 64000, 480, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + // Run Opus with 20 ms frame size. + RegisterSendCodec('A', codec_opus, 48000, 64000, 480 * 2, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + // Run Opus with 40 ms frame size. + RegisterSendCodec('A', codec_opus, 48000, 64000, 480 * 4, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + // Run Opus with 60 ms frame size. + RegisterSendCodec('A', codec_opus, 48000, 64000, 480 * 6, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + // Run Opus with 20 ms frame size and different bitrates. + RegisterSendCodec('A', codec_opus, 48000, 40000, 960, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_opus, 48000, 510000, 960, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif + // + // Test Mono-To-Stereo for all codecs. + // + audio_channels = 1; + codec_channels = 2; + + test_cntr_++; + channel_a2b_->set_codec_mode(kStereo); + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + channel_a2b_->set_codec_mode(kStereo); + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 320, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#ifdef PCMA_AND_PCMU + test_cntr_++; + channel_a2b_->set_codec_mode(kStereo); + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif +#ifdef WEBRTC_CODEC_OPUS + // Keep encode and decode in stereo. + test_cntr_++; + channel_a2b_->set_codec_mode(kStereo); + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_opus, 48000, 64000, 960, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + + // Encode in mono, decode in stereo mode. + RegisterSendCodec('A', codec_opus, 48000, 64000, 960, 1); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif + + // + // Test Stereo-To-Mono for all codecs. + // + audio_channels = 2; + codec_channels = 1; + channel_a2b_->set_codec_mode(kMono); + + // Run stereo audio and mono codec. + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_g722, 16000, 64000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 8000, 128000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 16000, 256000, 160, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_l16, 32000, 512000, 320, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#ifdef PCMA_AND_PCMU + test_cntr_++; + OpenOutFile(test_cntr_); + RegisterSendCodec('A', codec_pcmu, 8000, 64000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + RegisterSendCodec('A', codec_pcma, 8000, 64000, 80, codec_channels); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif +#ifdef WEBRTC_CODEC_OPUS + test_cntr_++; + OpenOutFile(test_cntr_); + // Encode and decode in mono. + RegisterSendCodec('A', codec_opus, 48000, 32000, 960, codec_channels); + acm_b_->SetReceiveCodecs({{120, {"OPUS", 48000, 2}}}); + Run(channel_a2b_, audio_channels, codec_channels); + + // Encode in stereo, decode in mono. + RegisterSendCodec('A', codec_opus, 48000, 32000, 960, 2); + Run(channel_a2b_, audio_channels, codec_channels); + + out_file_.Close(); + + // Test switching between decoding mono and stereo for Opus. + + // Decode in mono. + test_cntr_++; + OpenOutFile(test_cntr_); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); + // Decode in stereo. + test_cntr_++; + OpenOutFile(test_cntr_); + acm_b_->SetReceiveCodecs({{120, {"OPUS", 48000, 2, {{"stereo", "1"}}}}}); + Run(channel_a2b_, audio_channels, 2); + out_file_.Close(); + // Decode in mono. + test_cntr_++; + OpenOutFile(test_cntr_); + acm_b_->SetReceiveCodecs({{120, {"OPUS", 48000, 2}}}); + Run(channel_a2b_, audio_channels, codec_channels); + out_file_.Close(); +#endif + + // Delete the file pointers. + delete in_file_stereo_; + delete in_file_mono_; +} + +// Register Codec to use in the test +// +// Input: side - which ACM to use, 'A' or 'B' +// codec_name - name to use when register the codec +// sampling_freq_hz - sampling frequency in Herz +// rate - bitrate in bytes +// pack_size - packet size in samples +// channels - number of channels; 1 for mono, 2 for stereo +void TestStereo::RegisterSendCodec(char side, + char* codec_name, + int32_t sampling_freq_hz, + int rate, + int pack_size, + int channels) { + // Store packet size in samples, used to validate the received packet + pack_size_samp_ = pack_size; + + // Store the expected packet size in bytes, used to validate the received + // packet. Add 0.875 to always round up to a whole byte. + pack_size_bytes_ = (uint16_t)(static_cast(pack_size * rate) / + static_cast(sampling_freq_hz * 8) + + 0.875); + + // Set pointer to the ACM where to register the codec + AudioCodingModule* my_acm = NULL; + switch (side) { + case 'A': { + my_acm = acm_a_.get(); + break; + } + case 'B': { + my_acm = acm_b_.get(); + break; + } + default: + break; + } + ASSERT_TRUE(my_acm != NULL); + + auto encoder_factory = CreateBuiltinAudioEncoderFactory(); + const int clockrate_hz = absl::EqualsIgnoreCase(codec_name, "g722") + ? sampling_freq_hz / 2 + : sampling_freq_hz; + const std::string ptime = rtc::ToString(rtc::CheckedDivExact( + pack_size, rtc::CheckedDivExact(sampling_freq_hz, 1000))); + SdpAudioFormat::Parameters params = {{"ptime", ptime}}; + RTC_CHECK(channels == 1 || channels == 2); + if (absl::EqualsIgnoreCase(codec_name, "opus")) { + if (channels == 2) { + params["stereo"] = "1"; + } + channels = 2; + params["maxaveragebitrate"] = rtc::ToString(rate); + } + constexpr int payload_type = 17; + auto encoder = encoder_factory->MakeAudioEncoder( + payload_type, SdpAudioFormat(codec_name, clockrate_hz, channels, params), + absl::nullopt); + EXPECT_NE(nullptr, encoder); + my_acm->SetEncoder(std::move(encoder)); + + send_codec_name_ = codec_name; +} + +void TestStereo::Run(TestPackStereo* channel, + int in_channels, + int out_channels, + int percent_loss) { + AudioFrame audio_frame; + + int32_t out_freq_hz_b = out_file_.SamplingFrequency(); + uint16_t rec_size; + uint32_t time_stamp_diff; + channel->reset_payload_size(); + int error_count = 0; + int variable_bytes = 0; + int variable_packets = 0; + // Set test length to 500 ms (50 blocks of 10 ms each). + in_file_mono_->SetNum10MsBlocksToRead(50); + in_file_stereo_->SetNum10MsBlocksToRead(50); + // Fast-forward 1 second (100 blocks) since the files start with silence. + in_file_stereo_->FastForward(100); + in_file_mono_->FastForward(100); + + while (true) { + // Simulate packet loss by setting `packet_loss_` to "true" in + // `percent_loss` percent of the loops. + if (percent_loss > 0) { + if (counter_ == floor((100 / percent_loss) + 0.5)) { + counter_ = 0; + channel->set_lost_packet(true); + } else { + channel->set_lost_packet(false); + } + counter_++; + } + + // Add 10 msec to ACM + if (in_channels == 1) { + if (in_file_mono_->EndOfFile()) { + break; + } + in_file_mono_->Read10MsData(audio_frame); + } else { + if (in_file_stereo_->EndOfFile()) { + break; + } + in_file_stereo_->Read10MsData(audio_frame); + } + EXPECT_GE(acm_a_->Add10MsData(audio_frame), 0); + + // Verify that the received packet size matches the settings. + rec_size = channel->payload_size(); + if ((0 < rec_size) & (rec_size < 65535)) { + if (strcmp(send_codec_name_, "opus") == 0) { + // Opus is a variable rate codec, hence calculate the average packet + // size, and later make sure the average is in the right range. + variable_bytes += rec_size; + variable_packets++; + } else { + // For fixed rate codecs, check that packet size is correct. + if ((rec_size != pack_size_bytes_ * out_channels) && + (pack_size_bytes_ < 65535)) { + error_count++; + } + } + // Verify that the timestamp is updated with expected length + time_stamp_diff = channel->timestamp_diff(); + if ((counter_ > 10) && (time_stamp_diff != pack_size_samp_)) { + error_count++; + } + } + + // Run receive side of ACM + bool muted; + EXPECT_EQ(0, acm_b_->PlayoutData10Ms(out_freq_hz_b, &audio_frame, &muted)); + ASSERT_FALSE(muted); + + // Write output speech to file + out_file_.Write10MsData( + audio_frame.data(), + audio_frame.samples_per_channel_ * audio_frame.num_channels_); + } + + EXPECT_EQ(0, error_count); + + // Check that packet size is in the right range for variable rate codecs, + // such as Opus. + if (variable_packets > 0) { + variable_bytes /= variable_packets; + EXPECT_NEAR(variable_bytes, pack_size_bytes_, 18); + } + + if (in_file_mono_->EndOfFile()) { + in_file_mono_->Rewind(); + } + if (in_file_stereo_->EndOfFile()) { + in_file_stereo_->Rewind(); + } + // Reset in case we ended with a lost packet + channel->set_lost_packet(false); +} + +void TestStereo::OpenOutFile(int16_t test_number) { + std::string file_name; + rtc::StringBuilder file_stream; + file_stream << webrtc::test::OutputPath() << "teststereo_out_" << test_number + << ".pcm"; + file_name = file_stream.str(); + out_file_.Open(file_name, 32000, "wb"); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestStereo.h b/third_party/libwebrtc/modules/audio_coding/test/TestStereo.h new file mode 100644 index 0000000000..4c50a4b555 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TestStereo.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TESTSTEREO_H_ +#define MODULES_AUDIO_CODING_TEST_TESTSTEREO_H_ + +#include + +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/PCMFile.h" + +#define PCMA_AND_PCMU + +namespace webrtc { + +enum StereoMonoMode { kNotSet, kMono, kStereo }; + +class TestPackStereo : public AudioPacketizationCallback { + public: + TestPackStereo(); + ~TestPackStereo(); + + void RegisterReceiverACM(AudioCodingModule* acm); + + int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms) override; + + uint16_t payload_size(); + uint32_t timestamp_diff(); + void reset_payload_size(); + void set_codec_mode(StereoMonoMode mode); + void set_lost_packet(bool lost); + + private: + AudioCodingModule* receiver_acm_; + int16_t seq_no_; + uint32_t timestamp_diff_; + uint32_t last_in_timestamp_; + uint64_t total_bytes_; + int payload_size_; + StereoMonoMode codec_mode_; + // Simulate packet losses + bool lost_packet_; +}; + +class TestStereo { + public: + TestStereo(); + ~TestStereo(); + + void Perform(); + + private: + // The default value of '-1' indicates that the registration is based only on + // codec name and a sampling frequncy matching is not required. This is useful + // for codecs which support several sampling frequency. + void RegisterSendCodec(char side, + char* codec_name, + int32_t samp_freq_hz, + int rate, + int pack_size, + int channels); + + void Run(TestPackStereo* channel, + int in_channels, + int out_channels, + int percent_loss = 0); + void OpenOutFile(int16_t test_number); + + std::unique_ptr acm_a_; + std::unique_ptr acm_b_; + + TestPackStereo* channel_a2b_; + + PCMFile* in_file_stereo_; + PCMFile* in_file_mono_; + PCMFile out_file_; + int16_t test_cntr_; + uint16_t pack_size_samp_; + uint16_t pack_size_bytes_; + int counter_; + char* send_codec_name_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TESTSTEREO_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestVADDTX.cc b/third_party/libwebrtc/modules/audio_coding/test/TestVADDTX.cc new file mode 100644 index 0000000000..de26cafb68 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TestVADDTX.cc @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/TestVADDTX.h" + +#include + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "api/audio_codecs/audio_decoder_factory_template.h" +#include "api/audio_codecs/audio_encoder_factory_template.h" +#include "api/audio_codecs/ilbc/audio_decoder_ilbc.h" +#include "api/audio_codecs/ilbc/audio_encoder_ilbc.h" +#include "api/audio_codecs/opus/audio_decoder_opus.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +MonitoringAudioPacketizationCallback::MonitoringAudioPacketizationCallback( + AudioPacketizationCallback* next) + : next_(next) { + ResetStatistics(); +} + +int32_t MonitoringAudioPacketizationCallback::SendData( + AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + int64_t absolute_capture_timestamp_ms) { + counter_[static_cast(frame_type)]++; + return next_->SendData(frame_type, payload_type, timestamp, payload_data, + payload_len_bytes, absolute_capture_timestamp_ms); +} + +void MonitoringAudioPacketizationCallback::PrintStatistics() { + printf("\n"); + printf("kEmptyFrame %u\n", + counter_[static_cast(AudioFrameType::kEmptyFrame)]); + printf("kAudioFrameSpeech %u\n", + counter_[static_cast(AudioFrameType::kAudioFrameSpeech)]); + printf("kAudioFrameCN %u\n", + counter_[static_cast(AudioFrameType::kAudioFrameCN)]); + printf("\n\n"); +} + +void MonitoringAudioPacketizationCallback::ResetStatistics() { + memset(counter_, 0, sizeof(counter_)); +} + +void MonitoringAudioPacketizationCallback::GetStatistics(uint32_t* counter) { + memcpy(counter, counter_, sizeof(counter_)); +} + +TestVadDtx::TestVadDtx() + : encoder_factory_( + CreateAudioEncoderFactory()), + decoder_factory_( + CreateAudioDecoderFactory()), + acm_send_(AudioCodingModule::Create( + AudioCodingModule::Config(decoder_factory_))), + acm_receive_(AudioCodingModule::Create( + AudioCodingModule::Config(decoder_factory_))), + channel_(std::make_unique()), + packetization_callback_( + std::make_unique( + channel_.get())) { + EXPECT_EQ( + 0, acm_send_->RegisterTransportCallback(packetization_callback_.get())); + channel_->RegisterReceiverACM(acm_receive_.get()); +} + +bool TestVadDtx::RegisterCodec(const SdpAudioFormat& codec_format, + absl::optional vad_mode) { + constexpr int payload_type = 17, cn_payload_type = 117; + bool added_comfort_noise = false; + + auto encoder = encoder_factory_->MakeAudioEncoder(payload_type, codec_format, + absl::nullopt); + if (vad_mode.has_value() && + !absl::EqualsIgnoreCase(codec_format.name, "opus")) { + AudioEncoderCngConfig config; + config.speech_encoder = std::move(encoder); + config.num_channels = 1; + config.payload_type = cn_payload_type; + config.vad_mode = vad_mode.value(); + encoder = CreateComfortNoiseEncoder(std::move(config)); + added_comfort_noise = true; + } + channel_->SetIsStereo(encoder->NumChannels() > 1); + acm_send_->SetEncoder(std::move(encoder)); + + std::map receive_codecs = {{payload_type, codec_format}}; + acm_receive_->SetReceiveCodecs(receive_codecs); + + return added_comfort_noise; +} + +// Encoding a file and see if the numbers that various packets occur follow +// the expectation. +void TestVadDtx::Run(absl::string_view in_filename, + int frequency, + int channels, + absl::string_view out_filename, + bool append, + const int* expects) { + packetization_callback_->ResetStatistics(); + + PCMFile in_file; + in_file.Open(in_filename, frequency, "rb"); + in_file.ReadStereo(channels > 1); + // Set test length to 1000 ms (100 blocks of 10 ms each). + in_file.SetNum10MsBlocksToRead(100); + // Fast-forward both files 500 ms (50 blocks). The first second of the file is + // silence, but we want to keep half of that to test silence periods. + in_file.FastForward(50); + + PCMFile out_file; + if (append) { + out_file.Open(out_filename, kOutputFreqHz, "ab"); + } else { + out_file.Open(out_filename, kOutputFreqHz, "wb"); + } + + uint16_t frame_size_samples = in_file.PayloadLength10Ms(); + AudioFrame audio_frame; + while (!in_file.EndOfFile()) { + in_file.Read10MsData(audio_frame); + audio_frame.timestamp_ = time_stamp_; + time_stamp_ += frame_size_samples; + EXPECT_GE(acm_send_->Add10MsData(audio_frame), 0); + bool muted; + acm_receive_->PlayoutData10Ms(kOutputFreqHz, &audio_frame, &muted); + ASSERT_FALSE(muted); + out_file.Write10MsData(audio_frame); + } + + in_file.Close(); + out_file.Close(); + +#ifdef PRINT_STAT + packetization_callback_->PrintStatistics(); +#endif + + uint32_t stats[3]; + packetization_callback_->GetStatistics(stats); + packetization_callback_->ResetStatistics(); + + for (const auto& st : stats) { + int i = &st - stats; // Calculate the current position in stats. + switch (expects[i]) { + case 0: { + EXPECT_EQ(0u, st) << "stats[" << i << "] error."; + break; + } + case 1: { + EXPECT_GT(st, 0u) << "stats[" << i << "] error."; + break; + } + } + } +} + +// Following is the implementation of TestWebRtcVadDtx. +TestWebRtcVadDtx::TestWebRtcVadDtx() : output_file_num_(0) {} + +void TestWebRtcVadDtx::Perform() { + RunTestCases({"ILBC", 8000, 1}); + RunTestCases({"opus", 48000, 2}); +} + +// Test various configurations on VAD/DTX. +void TestWebRtcVadDtx::RunTestCases(const SdpAudioFormat& codec_format) { + Test(/*new_outfile=*/true, + /*expect_dtx_enabled=*/RegisterCodec(codec_format, absl::nullopt)); + + Test(/*new_outfile=*/false, + /*expect_dtx_enabled=*/RegisterCodec(codec_format, Vad::kVadAggressive)); + + Test(/*new_outfile=*/false, + /*expect_dtx_enabled=*/RegisterCodec(codec_format, Vad::kVadLowBitrate)); + + Test(/*new_outfile=*/false, /*expect_dtx_enabled=*/RegisterCodec( + codec_format, Vad::kVadVeryAggressive)); + + Test(/*new_outfile=*/false, + /*expect_dtx_enabled=*/RegisterCodec(codec_format, Vad::kVadNormal)); +} + +// Set the expectation and run the test. +void TestWebRtcVadDtx::Test(bool new_outfile, bool expect_dtx_enabled) { + int expects[] = {-1, 1, expect_dtx_enabled, 0, 0}; + if (new_outfile) { + output_file_num_++; + } + rtc::StringBuilder out_filename; + out_filename << webrtc::test::OutputPath() << "testWebRtcVadDtx_outFile_" + << output_file_num_ << ".pcm"; + Run(webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), 32000, 1, + out_filename.str(), !new_outfile, expects); +} + +// Following is the implementation of TestOpusDtx. +void TestOpusDtx::Perform() { + int expects[] = {0, 1, 0, 0, 0}; + + // Register Opus as send codec + std::string out_filename = + webrtc::test::OutputPath() + "testOpusDtx_outFile_mono.pcm"; + RegisterCodec({"opus", 48000, 2}, absl::nullopt); + acm_send_->ModifyEncoder([](std::unique_ptr* encoder_ptr) { + (*encoder_ptr)->SetDtx(false); + }); + + Run(webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), 32000, 1, + out_filename, false, expects); + + acm_send_->ModifyEncoder([](std::unique_ptr* encoder_ptr) { + (*encoder_ptr)->SetDtx(true); + }); + expects[static_cast(AudioFrameType::kEmptyFrame)] = 1; + expects[static_cast(AudioFrameType::kAudioFrameCN)] = 1; + Run(webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"), 32000, 1, + out_filename, true, expects); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestVADDTX.h b/third_party/libwebrtc/modules/audio_coding/test/TestVADDTX.h new file mode 100644 index 0000000000..d81ae28beb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TestVADDTX.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TESTVADDTX_H_ +#define MODULES_AUDIO_CODING_TEST_TESTVADDTX_H_ + +#include + +#include "absl/strings/string_view.h" +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "common_audio/vad/include/vad.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/test/Channel.h" + +namespace webrtc { + +// This class records the frame type, and delegates actual sending to the +// `next_` AudioPacketizationCallback. +class MonitoringAudioPacketizationCallback : public AudioPacketizationCallback { + public: + explicit MonitoringAudioPacketizationCallback( + AudioPacketizationCallback* next); + + int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_len_bytes, + int64_t absolute_capture_timestamp_ms) override; + + void PrintStatistics(); + void ResetStatistics(); + void GetStatistics(uint32_t* stats); + + private: + // 0 - kEmptyFrame + // 1 - kAudioFrameSpeech + // 2 - kAudioFrameCN + uint32_t counter_[3]; + AudioPacketizationCallback* const next_; +}; + +// TestVadDtx is to verify that VAD/DTX perform as they should. It runs through +// an audio file and check if the occurrence of various packet types follows +// expectation. TestVadDtx needs its derived class to implement the Perform() +// to put the test together. +class TestVadDtx { + public: + static const int kOutputFreqHz = 16000; + + TestVadDtx(); + + protected: + // Returns true iff CN was added. + bool RegisterCodec(const SdpAudioFormat& codec_format, + absl::optional vad_mode); + + // Encoding a file and see if the numbers that various packets occur follow + // the expectation. Saves result to a file. + // expects[x] means + // -1 : do not care, + // 0 : there have been no packets of type `x`, + // 1 : there have been packets of type `x`, + // with `x` indicates the following packet types + // 0 - kEmptyFrame + // 1 - kAudioFrameSpeech + // 2 - kAudioFrameCN + void Run(absl::string_view in_filename, + int frequency, + int channels, + absl::string_view out_filename, + bool append, + const int* expects); + + const rtc::scoped_refptr encoder_factory_; + const rtc::scoped_refptr decoder_factory_; + std::unique_ptr acm_send_; + std::unique_ptr acm_receive_; + std::unique_ptr channel_; + std::unique_ptr packetization_callback_; + uint32_t time_stamp_ = 0x12345678; +}; + +// TestWebRtcVadDtx is to verify that the WebRTC VAD/DTX perform as they should. +class TestWebRtcVadDtx final : public TestVadDtx { + public: + TestWebRtcVadDtx(); + + void Perform(); + + private: + void RunTestCases(const SdpAudioFormat& codec_format); + void Test(bool new_outfile, bool expect_dtx_enabled); + + int output_file_num_; +}; + +// TestOpusDtx is to verify that the Opus DTX performs as it should. +class TestOpusDtx final : public TestVadDtx { + public: + void Perform(); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TESTVADDTX_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/Tester.cc b/third_party/libwebrtc/modules/audio_coding/test/Tester.cc new file mode 100644 index 0000000000..7612aa43a3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/Tester.cc @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include +#include + +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/EncodeDecodeTest.h" +#include "modules/audio_coding/test/PacketLossTest.h" +#include "modules/audio_coding/test/TestAllCodecs.h" +#include "modules/audio_coding/test/TestRedFec.h" +#include "modules/audio_coding/test/TestStereo.h" +#include "modules/audio_coding/test/TestVADDTX.h" +#include "modules/audio_coding/test/TwoWayCommunication.h" +#include "modules/audio_coding/test/opus_test.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +TEST(AudioCodingModuleTest, TestAllCodecs) { + webrtc::TestAllCodecs().Perform(); +} + +#if defined(WEBRTC_ANDROID) +TEST(AudioCodingModuleTest, DISABLED_TestEncodeDecode) { +#else +TEST(AudioCodingModuleTest, TestEncodeDecode) { +#endif + webrtc::EncodeDecodeTest().Perform(); +} + +TEST(AudioCodingModuleTest, TestRedFec) { + webrtc::TestRedFec().Perform(); +} + +// Disabled on ios as flaky, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestStereo) { +#else +TEST(AudioCodingModuleTest, TestStereo) { +#endif + webrtc::TestStereo().Perform(); +} + +TEST(AudioCodingModuleTest, TestWebRtcVadDtx) { + webrtc::TestWebRtcVadDtx().Perform(); +} + +TEST(AudioCodingModuleTest, TestOpusDtx) { + webrtc::TestOpusDtx().Perform(); +} + +// Disabled on ios as flaky, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestOpus) { +#else +TEST(AudioCodingModuleTest, TestOpus) { +#endif + webrtc::OpusTest().Perform(); +} + +TEST(AudioCodingModuleTest, TestPacketLoss) { + webrtc::PacketLossTest(1, 10, 10, 1).Perform(); +} + +TEST(AudioCodingModuleTest, TestPacketLossBurst) { + webrtc::PacketLossTest(1, 10, 10, 2).Perform(); +} + +// Disabled on ios as flake, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestPacketLossStereo) { +#else +TEST(AudioCodingModuleTest, TestPacketLossStereo) { +#endif + webrtc::PacketLossTest(2, 10, 10, 1).Perform(); +} + +// Disabled on ios as flake, see https://crbug.com/webrtc/7057 +#if defined(WEBRTC_IOS) +TEST(AudioCodingModuleTest, DISABLED_TestPacketLossStereoBurst) { +#else +TEST(AudioCodingModuleTest, TestPacketLossStereoBurst) { +#endif + webrtc::PacketLossTest(2, 10, 10, 2).Perform(); +} + +// The full API test is too long to run automatically on bots, but can be used +// for offline testing. User interaction is needed. +#ifdef ACM_TEST_FULL_API +TEST(AudioCodingModuleTest, TestAPI) { + webrtc::APITest().Perform(); +} +#endif diff --git a/third_party/libwebrtc/modules/audio_coding/test/TwoWayCommunication.cc b/third_party/libwebrtc/modules/audio_coding/test/TwoWayCommunication.cc new file mode 100644 index 0000000000..b42415a21a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TwoWayCommunication.cc @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "TwoWayCommunication.h" + +#include +#include + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +#define MAX_FILE_NAME_LENGTH_BYTE 500 + +TwoWayCommunication::TwoWayCommunication() + : _acmA(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))), + _acmRefA(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))) { + AudioCodingModule::Config config; + // The clicks will be more obvious if time-stretching is not allowed. + // TODO(henrik.lundin) Really? + config.neteq_config.for_test_no_time_stretching = true; + config.decoder_factory = CreateBuiltinAudioDecoderFactory(); + _acmB.reset(AudioCodingModule::Create(config)); + _acmRefB.reset(AudioCodingModule::Create(config)); +} + +TwoWayCommunication::~TwoWayCommunication() { + delete _channel_A2B; + delete _channel_B2A; + delete _channelRef_A2B; + delete _channelRef_B2A; + _inFileA.Close(); + _inFileB.Close(); + _outFileA.Close(); + _outFileB.Close(); + _outFileRefA.Close(); + _outFileRefB.Close(); +} + +void TwoWayCommunication::SetUpAutotest( + AudioEncoderFactory* const encoder_factory, + const SdpAudioFormat& format1, + const int payload_type1, + const SdpAudioFormat& format2, + const int payload_type2) { + //--- Set A codecs + _acmA->SetEncoder( + encoder_factory->MakeAudioEncoder(payload_type1, format1, absl::nullopt)); + _acmA->SetReceiveCodecs({{payload_type2, format2}}); + + //--- Set ref-A codecs + _acmRefA->SetEncoder( + encoder_factory->MakeAudioEncoder(payload_type1, format1, absl::nullopt)); + _acmRefA->SetReceiveCodecs({{payload_type2, format2}}); + + //--- Set B codecs + _acmB->SetEncoder( + encoder_factory->MakeAudioEncoder(payload_type2, format2, absl::nullopt)); + _acmB->SetReceiveCodecs({{payload_type1, format1}}); + + //--- Set ref-B codecs + _acmRefB->SetEncoder( + encoder_factory->MakeAudioEncoder(payload_type2, format2, absl::nullopt)); + _acmRefB->SetReceiveCodecs({{payload_type1, format1}}); + + uint16_t frequencyHz; + + //--- Input A and B + std::string in_file_name = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + frequencyHz = 16000; + _inFileA.Open(in_file_name, frequencyHz, "rb"); + _inFileB.Open(in_file_name, frequencyHz, "rb"); + + //--- Output A + std::string output_file_a = webrtc::test::OutputPath() + "outAutotestA.pcm"; + frequencyHz = 16000; + _outFileA.Open(output_file_a, frequencyHz, "wb"); + std::string output_ref_file_a = + webrtc::test::OutputPath() + "ref_outAutotestA.pcm"; + _outFileRefA.Open(output_ref_file_a, frequencyHz, "wb"); + + //--- Output B + std::string output_file_b = webrtc::test::OutputPath() + "outAutotestB.pcm"; + frequencyHz = 16000; + _outFileB.Open(output_file_b, frequencyHz, "wb"); + std::string output_ref_file_b = + webrtc::test::OutputPath() + "ref_outAutotestB.pcm"; + _outFileRefB.Open(output_ref_file_b, frequencyHz, "wb"); + + //--- Set A-to-B channel + _channel_A2B = new Channel; + _acmA->RegisterTransportCallback(_channel_A2B); + _channel_A2B->RegisterReceiverACM(_acmB.get()); + //--- Do the same for the reference + _channelRef_A2B = new Channel; + _acmRefA->RegisterTransportCallback(_channelRef_A2B); + _channelRef_A2B->RegisterReceiverACM(_acmRefB.get()); + + //--- Set B-to-A channel + _channel_B2A = new Channel; + _acmB->RegisterTransportCallback(_channel_B2A); + _channel_B2A->RegisterReceiverACM(_acmA.get()); + //--- Do the same for reference + _channelRef_B2A = new Channel; + _acmRefB->RegisterTransportCallback(_channelRef_B2A); + _channelRef_B2A->RegisterReceiverACM(_acmRefA.get()); +} + +void TwoWayCommunication::Perform() { + const SdpAudioFormat format1("ISAC", 16000, 1); + const SdpAudioFormat format2("L16", 8000, 1); + constexpr int payload_type1 = 17, payload_type2 = 18; + + auto encoder_factory = CreateBuiltinAudioEncoderFactory(); + + SetUpAutotest(encoder_factory.get(), format1, payload_type1, format2, + payload_type2); + + unsigned int msecPassed = 0; + unsigned int secPassed = 0; + + int32_t outFreqHzA = _outFileA.SamplingFrequency(); + int32_t outFreqHzB = _outFileB.SamplingFrequency(); + + AudioFrame audioFrame; + + // In the following loop we tests that the code can handle misuse of the APIs. + // In the middle of a session with data flowing between two sides, called A + // and B, APIs will be called, and the code should continue to run, and be + // able to recover. + while (!_inFileA.EndOfFile() && !_inFileB.EndOfFile()) { + msecPassed += 10; + EXPECT_GT(_inFileA.Read10MsData(audioFrame), 0); + EXPECT_GE(_acmA->Add10MsData(audioFrame), 0); + EXPECT_GE(_acmRefA->Add10MsData(audioFrame), 0); + + EXPECT_GT(_inFileB.Read10MsData(audioFrame), 0); + + EXPECT_GE(_acmB->Add10MsData(audioFrame), 0); + EXPECT_GE(_acmRefB->Add10MsData(audioFrame), 0); + bool muted; + EXPECT_EQ(0, _acmA->PlayoutData10Ms(outFreqHzA, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileA.Write10MsData(audioFrame); + EXPECT_EQ(0, _acmRefA->PlayoutData10Ms(outFreqHzA, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileRefA.Write10MsData(audioFrame); + EXPECT_EQ(0, _acmB->PlayoutData10Ms(outFreqHzB, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileB.Write10MsData(audioFrame); + EXPECT_EQ(0, _acmRefB->PlayoutData10Ms(outFreqHzB, &audioFrame, &muted)); + ASSERT_FALSE(muted); + _outFileRefB.Write10MsData(audioFrame); + + // Update time counters each time a second of data has passed. + if (msecPassed >= 1000) { + msecPassed = 0; + secPassed++; + } + // Re-register send codec on side B. + if (((secPassed % 5) == 4) && (msecPassed >= 990)) { + _acmB->SetEncoder(encoder_factory->MakeAudioEncoder( + payload_type2, format2, absl::nullopt)); + } + // Initialize receiver on side A. + if (((secPassed % 7) == 6) && (msecPassed == 0)) + EXPECT_EQ(0, _acmA->InitializeReceiver()); + // Re-register codec on side A. + if (((secPassed % 7) == 6) && (msecPassed >= 990)) { + _acmA->SetReceiveCodecs({{payload_type2, format2}}); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/TwoWayCommunication.h b/third_party/libwebrtc/modules/audio_coding/test/TwoWayCommunication.h new file mode 100644 index 0000000000..b7eb9e5583 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/TwoWayCommunication.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_TWOWAYCOMMUNICATION_H_ +#define MODULES_AUDIO_CODING_TEST_TWOWAYCOMMUNICATION_H_ + +#include + +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/audio_codecs/audio_format.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_coding/test/Channel.h" +#include "modules/audio_coding/test/PCMFile.h" + +namespace webrtc { + +class TwoWayCommunication { + public: + TwoWayCommunication(); + ~TwoWayCommunication(); + + void Perform(); + + private: + void SetUpAutotest(AudioEncoderFactory* const encoder_factory, + const SdpAudioFormat& format1, + int payload_type1, + const SdpAudioFormat& format2, + int payload_type2); + + std::unique_ptr _acmA; + std::unique_ptr _acmB; + + std::unique_ptr _acmRefA; + std::unique_ptr _acmRefB; + + Channel* _channel_A2B; + Channel* _channel_B2A; + + Channel* _channelRef_A2B; + Channel* _channelRef_B2A; + + PCMFile _inFileA; + PCMFile _inFileB; + + PCMFile _outFileA; + PCMFile _outFileB; + + PCMFile _outFileRefA; + PCMFile _outFileRefB; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_TWOWAYCOMMUNICATION_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/opus_test.cc b/third_party/libwebrtc/modules/audio_coding/test/opus_test.cc new file mode 100644 index 0000000000..6822bc3d72 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/opus_test.cc @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/test/opus_test.h" + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/audio_coding/test/TestStereo.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +OpusTest::OpusTest() + : acm_receiver_(AudioCodingModule::Create( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory()))), + channel_a2b_(NULL), + counter_(0), + payload_type_(255), + rtp_timestamp_(0) {} + +OpusTest::~OpusTest() { + if (channel_a2b_ != NULL) { + delete channel_a2b_; + channel_a2b_ = NULL; + } + if (opus_mono_encoder_ != NULL) { + WebRtcOpus_EncoderFree(opus_mono_encoder_); + opus_mono_encoder_ = NULL; + } + if (opus_stereo_encoder_ != NULL) { + WebRtcOpus_EncoderFree(opus_stereo_encoder_); + opus_stereo_encoder_ = NULL; + } + if (opus_mono_decoder_ != NULL) { + WebRtcOpus_DecoderFree(opus_mono_decoder_); + opus_mono_decoder_ = NULL; + } + if (opus_stereo_decoder_ != NULL) { + WebRtcOpus_DecoderFree(opus_stereo_decoder_); + opus_stereo_decoder_ = NULL; + } +} + +void OpusTest::Perform() { +#ifndef WEBRTC_CODEC_OPUS + // Opus isn't defined, exit. + return; +#else + uint16_t frequency_hz; + size_t audio_channels; + int16_t test_cntr = 0; + + // Open both mono and stereo test files in 32 kHz. + const std::string file_name_stereo = + webrtc::test::ResourcePath("audio_coding/teststereo32kHz", "pcm"); + const std::string file_name_mono = + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm"); + frequency_hz = 32000; + in_file_stereo_.Open(file_name_stereo, frequency_hz, "rb"); + in_file_stereo_.ReadStereo(true); + in_file_mono_.Open(file_name_mono, frequency_hz, "rb"); + in_file_mono_.ReadStereo(false); + + // Create Opus encoders for mono and stereo. + ASSERT_GT(WebRtcOpus_EncoderCreate(&opus_mono_encoder_, 1, 0, 48000), -1); + ASSERT_GT(WebRtcOpus_EncoderCreate(&opus_stereo_encoder_, 2, 1, 48000), -1); + + // Create Opus decoders for mono and stereo for stand-alone testing of Opus. + ASSERT_GT(WebRtcOpus_DecoderCreate(&opus_mono_decoder_, 1, 48000), -1); + ASSERT_GT(WebRtcOpus_DecoderCreate(&opus_stereo_decoder_, 2, 48000), -1); + WebRtcOpus_DecoderInit(opus_mono_decoder_); + WebRtcOpus_DecoderInit(opus_stereo_decoder_); + + ASSERT_TRUE(acm_receiver_.get() != NULL); + EXPECT_EQ(0, acm_receiver_->InitializeReceiver()); + + // Register Opus stereo as receiving codec. + constexpr int kOpusPayloadType = 120; + const SdpAudioFormat kOpusFormatStereo("opus", 48000, 2, {{"stereo", "1"}}); + payload_type_ = kOpusPayloadType; + acm_receiver_->SetReceiveCodecs({{kOpusPayloadType, kOpusFormatStereo}}); + + // Create and connect the channel. + channel_a2b_ = new TestPackStereo; + channel_a2b_->RegisterReceiverACM(acm_receiver_.get()); + + // + // Test Stereo. + // + + channel_a2b_->set_codec_mode(kStereo); + audio_channels = 2; + test_cntr++; + OpenOutFile(test_cntr); + + // Run Opus with 2.5 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 120); + + // Run Opus with 5 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 240); + + // Run Opus with 10 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 480); + + // Run Opus with 20 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 960); + + // Run Opus with 40 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 1920); + + // Run Opus with 60 ms frame size. + Run(channel_a2b_, audio_channels, 64000, 2880); + + out_file_.Close(); + out_file_standalone_.Close(); + + // + // Test Opus stereo with packet-losses. + // + + test_cntr++; + OpenOutFile(test_cntr); + + // Run Opus with 20 ms frame size, 1% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 1); + + // Run Opus with 20 ms frame size, 5% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 5); + + // Run Opus with 20 ms frame size, 10% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 10); + + out_file_.Close(); + out_file_standalone_.Close(); + + // + // Test Mono. + // + channel_a2b_->set_codec_mode(kMono); + audio_channels = 1; + test_cntr++; + OpenOutFile(test_cntr); + + // Register Opus mono as receiving codec. + const SdpAudioFormat kOpusFormatMono("opus", 48000, 2); + acm_receiver_->SetReceiveCodecs({{kOpusPayloadType, kOpusFormatMono}}); + + // Run Opus with 2.5 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 120); + + // Run Opus with 5 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 240); + + // Run Opus with 10 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 480); + + // Run Opus with 20 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 960); + + // Run Opus with 40 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 1920); + + // Run Opus with 60 ms frame size. + Run(channel_a2b_, audio_channels, 32000, 2880); + + out_file_.Close(); + out_file_standalone_.Close(); + + // + // Test Opus mono with packet-losses. + // + test_cntr++; + OpenOutFile(test_cntr); + + // Run Opus with 20 ms frame size, 1% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 1); + + // Run Opus with 20 ms frame size, 5% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 5); + + // Run Opus with 20 ms frame size, 10% packet loss. + Run(channel_a2b_, audio_channels, 64000, 960, 10); + + // Close the files. + in_file_stereo_.Close(); + in_file_mono_.Close(); + out_file_.Close(); + out_file_standalone_.Close(); +#endif +} + +void OpusTest::Run(TestPackStereo* channel, + size_t channels, + int bitrate, + size_t frame_length, + int percent_loss) { + AudioFrame audio_frame; + int32_t out_freq_hz_b = out_file_.SamplingFrequency(); + const size_t kBufferSizeSamples = 480 * 12 * 2; // 120 ms stereo audio. + int16_t audio[kBufferSizeSamples]; + int16_t out_audio[kBufferSizeSamples]; + int16_t audio_type; + size_t written_samples = 0; + size_t read_samples = 0; + size_t decoded_samples = 0; + bool first_packet = true; + uint32_t start_time_stamp = 0; + + channel->reset_payload_size(); + counter_ = 0; + + // Set encoder rate. + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_mono_encoder_, bitrate)); + EXPECT_EQ(0, WebRtcOpus_SetBitRate(opus_stereo_encoder_, bitrate)); + +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_ARCH_ARM) + // If we are on Android, iOS and/or ARM, use a lower complexity setting as + // default. + const int kOpusComplexity5 = 5; + EXPECT_EQ(0, WebRtcOpus_SetComplexity(opus_mono_encoder_, kOpusComplexity5)); + EXPECT_EQ(0, + WebRtcOpus_SetComplexity(opus_stereo_encoder_, kOpusComplexity5)); +#endif + + // Fast-forward 1 second (100 blocks) since the files start with silence. + in_file_stereo_.FastForward(100); + in_file_mono_.FastForward(100); + + // Limit the runtime to 1000 blocks of 10 ms each. + for (size_t audio_length = 0; audio_length < 1000; audio_length += 10) { + bool lost_packet = false; + + // Get 10 msec of audio. + if (channels == 1) { + if (in_file_mono_.EndOfFile()) { + break; + } + in_file_mono_.Read10MsData(audio_frame); + } else { + if (in_file_stereo_.EndOfFile()) { + break; + } + in_file_stereo_.Read10MsData(audio_frame); + } + + // If input audio is sampled at 32 kHz, resampling to 48 kHz is required. + EXPECT_EQ(480, resampler_.Resample10Msec( + audio_frame.data(), audio_frame.sample_rate_hz_, 48000, + channels, kBufferSizeSamples - written_samples, + &audio[written_samples])); + written_samples += 480 * channels; + + // Sometimes we need to loop over the audio vector to produce the right + // number of packets. + size_t loop_encode = + (written_samples - read_samples) / (channels * frame_length); + + if (loop_encode > 0) { + const size_t kMaxBytes = 1000; // Maximum number of bytes for one packet. + size_t bitstream_len_byte; + uint8_t bitstream[kMaxBytes]; + for (size_t i = 0; i < loop_encode; i++) { + int bitstream_len_byte_int = WebRtcOpus_Encode( + (channels == 1) ? opus_mono_encoder_ : opus_stereo_encoder_, + &audio[read_samples], frame_length, kMaxBytes, bitstream); + ASSERT_GE(bitstream_len_byte_int, 0); + bitstream_len_byte = static_cast(bitstream_len_byte_int); + + // Simulate packet loss by setting `packet_loss_` to "true" in + // `percent_loss` percent of the loops. + // TODO(tlegrand): Move handling of loss simulation to TestPackStereo. + if (percent_loss > 0) { + if (counter_ == floor((100 / percent_loss) + 0.5)) { + counter_ = 0; + lost_packet = true; + channel->set_lost_packet(true); + } else { + lost_packet = false; + channel->set_lost_packet(false); + } + counter_++; + } + + // Run stand-alone Opus decoder, or decode PLC. + if (channels == 1) { + if (!lost_packet) { + decoded_samples += WebRtcOpus_Decode( + opus_mono_decoder_, bitstream, bitstream_len_byte, + &out_audio[decoded_samples * channels], &audio_type); + } else { + // Call decoder PLC. + constexpr int kPlcDurationMs = 10; + constexpr int kPlcSamples = 48 * kPlcDurationMs; + size_t total_plc_samples = 0; + while (total_plc_samples < frame_length) { + int ret = WebRtcOpus_Decode( + opus_mono_decoder_, NULL, 0, + &out_audio[decoded_samples * channels], &audio_type); + EXPECT_EQ(ret, kPlcSamples); + decoded_samples += ret; + total_plc_samples += ret; + } + EXPECT_EQ(total_plc_samples, frame_length); + } + } else { + if (!lost_packet) { + decoded_samples += WebRtcOpus_Decode( + opus_stereo_decoder_, bitstream, bitstream_len_byte, + &out_audio[decoded_samples * channels], &audio_type); + } else { + // Call decoder PLC. + constexpr int kPlcDurationMs = 10; + constexpr int kPlcSamples = 48 * kPlcDurationMs; + size_t total_plc_samples = 0; + while (total_plc_samples < frame_length) { + int ret = WebRtcOpus_Decode( + opus_stereo_decoder_, NULL, 0, + &out_audio[decoded_samples * channels], &audio_type); + EXPECT_EQ(ret, kPlcSamples); + decoded_samples += ret; + total_plc_samples += ret; + } + EXPECT_EQ(total_plc_samples, frame_length); + } + } + + // Send data to the channel. "channel" will handle the loss simulation. + channel->SendData(AudioFrameType::kAudioFrameSpeech, payload_type_, + rtp_timestamp_, bitstream, bitstream_len_byte, 0); + if (first_packet) { + first_packet = false; + start_time_stamp = rtp_timestamp_; + } + rtp_timestamp_ += static_cast(frame_length); + read_samples += frame_length * channels; + } + if (read_samples == written_samples) { + read_samples = 0; + written_samples = 0; + } + } + + // Run received side of ACM. + bool muted; + ASSERT_EQ( + 0, acm_receiver_->PlayoutData10Ms(out_freq_hz_b, &audio_frame, &muted)); + ASSERT_FALSE(muted); + + // Write output speech to file. + out_file_.Write10MsData( + audio_frame.data(), + audio_frame.samples_per_channel_ * audio_frame.num_channels_); + + // Write stand-alone speech to file. + out_file_standalone_.Write10MsData(out_audio, decoded_samples * channels); + + if (audio_frame.timestamp_ > start_time_stamp) { + // Number of channels should be the same for both stand-alone and + // ACM-decoding. + EXPECT_EQ(audio_frame.num_channels_, channels); + } + + decoded_samples = 0; + } + + if (in_file_mono_.EndOfFile()) { + in_file_mono_.Rewind(); + } + if (in_file_stereo_.EndOfFile()) { + in_file_stereo_.Rewind(); + } + // Reset in case we ended with a lost packet. + channel->set_lost_packet(false); +} + +void OpusTest::OpenOutFile(int test_number) { + std::string file_name; + std::stringstream file_stream; + file_stream << webrtc::test::OutputPath() << "opustest_out_" << test_number + << ".pcm"; + file_name = file_stream.str(); + out_file_.Open(file_name, 48000, "wb"); + file_stream.str(""); + file_name = file_stream.str(); + file_stream << webrtc::test::OutputPath() << "opusstandalone_out_" + << test_number << ".pcm"; + file_name = file_stream.str(); + out_file_standalone_.Open(file_name, 48000, "wb"); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/test/opus_test.h b/third_party/libwebrtc/modules/audio_coding/test/opus_test.h new file mode 100644 index 0000000000..c69f922adb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/opus_test.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_TEST_OPUS_TEST_H_ +#define MODULES_AUDIO_CODING_TEST_OPUS_TEST_H_ + +#include + +#include + +#include "modules/audio_coding/acm2/acm_resampler.h" +#include "modules/audio_coding/codecs/opus/opus_interface.h" +#include "modules/audio_coding/test/PCMFile.h" +#include "modules/audio_coding/test/TestStereo.h" + +namespace webrtc { + +class OpusTest { + public: + OpusTest(); + ~OpusTest(); + + void Perform(); + + private: + void Run(TestPackStereo* channel, + size_t channels, + int bitrate, + size_t frame_length, + int percent_loss = 0); + + void OpenOutFile(int test_number); + + std::unique_ptr acm_receiver_; + TestPackStereo* channel_a2b_; + PCMFile in_file_stereo_; + PCMFile in_file_mono_; + PCMFile out_file_; + PCMFile out_file_standalone_; + int counter_; + uint8_t payload_type_; + uint32_t rtp_timestamp_; + acm2::ACMResampler resampler_; + WebRtcOpusEncInst* opus_mono_encoder_; + WebRtcOpusEncInst* opus_stereo_encoder_; + WebRtcOpusDecInst* opus_mono_decoder_; + WebRtcOpusDecInst* opus_stereo_decoder_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_TEST_OPUS_TEST_H_ diff --git a/third_party/libwebrtc/modules/audio_coding/test/target_delay_unittest.cc b/third_party/libwebrtc/modules/audio_coding/test/target_delay_unittest.cc new file mode 100644 index 0000000000..5eccdcf8eb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/test/target_delay_unittest.cc @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/audio/audio_frame.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/rtp_headers.h" +#include "modules/audio_coding/acm2/acm_receiver.h" +#include "modules/audio_coding/codecs/pcm16b/pcm16b.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +class TargetDelayTest : public ::testing::Test { + protected: + TargetDelayTest() + : receiver_( + AudioCodingModule::Config(CreateBuiltinAudioDecoderFactory())) {} + + ~TargetDelayTest() {} + + void SetUp() { + constexpr int pltype = 108; + std::map receive_codecs = { + {pltype, {"L16", kSampleRateHz, 1}}}; + receiver_.SetCodecs(receive_codecs); + + rtp_header_.payloadType = pltype; + rtp_header_.timestamp = 0; + rtp_header_.ssrc = 0x12345678; + rtp_header_.markerBit = false; + rtp_header_.sequenceNumber = 0; + + int16_t audio[kFrameSizeSamples]; + const int kRange = 0x7FF; // 2047, easy for masking. + for (size_t n = 0; n < kFrameSizeSamples; ++n) + audio[n] = (rand() & kRange) - kRange / 2; + WebRtcPcm16b_Encode(audio, kFrameSizeSamples, payload_); + } + + void OutOfRangeInput() { + EXPECT_EQ(-1, SetMinimumDelay(-1)); + EXPECT_EQ(-1, SetMinimumDelay(10001)); + } + + void TargetDelayBufferMinMax() { + const int kTargetMinDelayMs = kNum10msPerFrame * 10; + ASSERT_EQ(0, SetMinimumDelay(kTargetMinDelayMs)); + for (int m = 0; m < 30; ++m) // Run enough iterations to fill the buffer. + Run(true); + int clean_optimal_delay = GetCurrentOptimalDelayMs(); + EXPECT_EQ(kTargetMinDelayMs, clean_optimal_delay); + + const int kTargetMaxDelayMs = 2 * (kNum10msPerFrame * 10); + ASSERT_EQ(0, SetMaximumDelay(kTargetMaxDelayMs)); + for (int n = 0; n < 30; ++n) // Run enough iterations to fill the buffer. + Run(false); + + int capped_optimal_delay = GetCurrentOptimalDelayMs(); + EXPECT_EQ(kTargetMaxDelayMs, capped_optimal_delay); + } + + private: + static const int kSampleRateHz = 16000; + static const int kNum10msPerFrame = 2; + static const size_t kFrameSizeSamples = 320; // 20 ms @ 16 kHz. + // payload-len = frame-samples * 2 bytes/sample. + static const int kPayloadLenBytes = 320 * 2; + // Inter-arrival time in number of packets in a jittery channel. One is no + // jitter. + static const int kInterarrivalJitterPacket = 2; + + void Push() { + rtp_header_.timestamp += kFrameSizeSamples; + rtp_header_.sequenceNumber++; + ASSERT_EQ(0, receiver_.InsertPacket(rtp_header_, + rtc::ArrayView( + payload_, kFrameSizeSamples * 2))); + } + + // Pull audio equivalent to the amount of audio in one RTP packet. + void Pull() { + AudioFrame frame; + bool muted; + for (int k = 0; k < kNum10msPerFrame; ++k) { // Pull one frame. + ASSERT_EQ(0, receiver_.GetAudio(-1, &frame, &muted)); + ASSERT_FALSE(muted); + // Had to use ASSERT_TRUE, ASSERT_EQ generated error. + ASSERT_TRUE(kSampleRateHz == frame.sample_rate_hz_); + ASSERT_EQ(1u, frame.num_channels_); + ASSERT_TRUE(kSampleRateHz / 100 == frame.samples_per_channel_); + } + } + + void Run(bool clean) { + for (int n = 0; n < 10; ++n) { + for (int m = 0; m < 5; ++m) { + Push(); + Pull(); + } + + if (!clean) { + for (int m = 0; m < 10; ++m) { // Long enough to trigger delay change. + Push(); + for (int n = 0; n < kInterarrivalJitterPacket; ++n) + Pull(); + } + } + } + } + + int SetMinimumDelay(int delay_ms) { + return receiver_.SetMinimumDelay(delay_ms); + } + + int SetMaximumDelay(int delay_ms) { + return receiver_.SetMaximumDelay(delay_ms); + } + + int GetCurrentOptimalDelayMs() { + NetworkStatistics stats; + receiver_.GetNetworkStatistics(&stats); + return stats.preferredBufferSize; + } + + acm2::AcmReceiver receiver_; + RTPHeader rtp_header_; + uint8_t payload_[kPayloadLenBytes]; +}; + +// Flaky on iOS: webrtc:7057. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_OutOfRangeInput DISABLED_OutOfRangeInput +#else +#define MAYBE_OutOfRangeInput OutOfRangeInput +#endif +TEST_F(TargetDelayTest, MAYBE_OutOfRangeInput) { + OutOfRangeInput(); +} + +// Flaky on iOS: webrtc:7057. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_TargetDelayBufferMinMax DISABLED_TargetDelayBufferMinMax +#else +#define MAYBE_TargetDelayBufferMinMax TargetDelayBufferMinMax +#endif +TEST_F(TargetDelayTest, MAYBE_TargetDelayBufferMinMax) { + TargetDelayBufferMinMax(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_coding/webrtc_cng_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/webrtc_cng_gn/moz.build new file mode 100644 index 0000000000..4a9c24069e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/webrtc_cng_gn/moz.build @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/cng/webrtc_cng.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("webrtc_cng_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/webrtc_multiopus_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/webrtc_multiopus_gn/moz.build new file mode 100644 index 0000000000..99ea47c13a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/webrtc_multiopus_gn/moz.build @@ -0,0 +1,230 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_CODEC_ILBC"] = True +DEFINES["WEBRTC_CODEC_OPUS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_OPUS_SUPPORT_120MS_PTIME"] = "1" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/media/libopus/include/", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_multi_channel_opus_impl.cc", + "/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("webrtc_multiopus_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/webrtc_opus_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/webrtc_opus_gn/moz.build new file mode 100644 index 0000000000..49fb552b0d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/webrtc_opus_gn/moz.build @@ -0,0 +1,237 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_CODEC_ILBC"] = True +DEFINES["WEBRTC_CODEC_OPUS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_OPUS_SUPPORT_120MS_PTIME"] = "1" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/media/libopus/include/", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_decoder_opus.cc", + "/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("webrtc_opus_gn") diff --git a/third_party/libwebrtc/modules/audio_coding/webrtc_opus_wrapper_gn/moz.build b/third_party/libwebrtc/modules/audio_coding/webrtc_opus_wrapper_gn/moz.build new file mode 100644 index 0000000000..20e1d37306 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_coding/webrtc_opus_wrapper_gn/moz.build @@ -0,0 +1,229 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_CODEC_ILBC"] = True +DEFINES["WEBRTC_CODEC_OPUS"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_OPUS_SUPPORT_120MS_PTIME"] = "1" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/media/libopus/include/", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_coding/codecs/opus/opus_interface.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("webrtc_opus_wrapper_gn") -- cgit v1.2.3